diff --git a/.github/workflows/images.yml b/.github/workflows/images.yml index e9c68b1..faf72f5 100644 --- a/.github/workflows/images.yml +++ b/.github/workflows/images.yml @@ -9,9 +9,19 @@ on: tag: type: string default: '' + publish_sdk_gateway: + type: boolean + default: false + workflow_dispatch: + inputs: + publish_sdk_gateway: + description: Publish the pinned MCP SDK v2 gateway used by FYRE comparisons + type: boolean + default: false jobs: build: + if: ${{ !inputs.publish_sdk_gateway }} name: Images (${{ matrix.arch }}) strategy: fail-fast: false @@ -108,7 +118,7 @@ jobs: publish: name: Publish image manifests - if: inputs.publish + if: inputs.publish && !inputs.publish_sdk_gateway needs: build runs-on: ubuntu-24.04 permissions: @@ -149,3 +159,96 @@ jobs: exit 1 fi done + + sdk-gateway: + name: SDK v2 gateway (${{ matrix.arch }}) + if: inputs.publish_sdk_gateway + strategy: + fail-fast: false + matrix: + include: + - runner: ubuntu-24.04 + arch: amd64 + - runner: ubuntu-24.04-arm + arch: arm64 + runs-on: ${{ matrix.runner }} + permissions: + contents: read + packages: write + env: + GATEWAY_REVISION: 33e2dd93a53a9cc2c5088b731822dfec4852fa2e + GATEWAY_TAG: sdk-v2-33e2dd93a + steps: + - name: Checkout benchmark repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: benchmark + persist-credentials: false + + - name: Checkout pinned gateway source + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: IBM/mcp-context-forge + ref: ${{ env.GATEWAY_REVISION }} + path: gateway + persist-credentials: false + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 + + - name: Log in to GHCR + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push pinned gateway + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 + with: + context: gateway + file: gateway/Containerfile + platforms: linux/${{ matrix.arch }} + push: true + tags: ghcr.io/contextforge-org/cf-integration-fixture:gateway-${{ env.GATEWAY_TAG }}-${{ matrix.arch }} + labels: | + org.opencontainers.image.revision=${{ env.GATEWAY_REVISION }} + org.opencontainers.image.source=https://github.com/contextforge-org/contextforge-dev-tools + cache-from: type=gha,scope=sdk-gateway-${{ matrix.arch }} + cache-to: type=gha,mode=max,scope=sdk-gateway-${{ matrix.arch }} + provenance: false + + publish-sdk-gateway: + name: Publish SDK v2 gateway manifest + if: inputs.publish_sdk_gateway + needs: sdk-gateway + runs-on: ubuntu-24.04 + permissions: + contents: read + packages: write + env: + GATEWAY_TAG: sdk-v2-33e2dd93a + steps: + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 + + - name: Log in to GHCR + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Publish multi-platform manifest + run: | + ref=ghcr.io/contextforge-org/cf-integration-fixture + docker buildx imagetools create \ + --tag "$ref:gateway-$GATEWAY_TAG" \ + "$ref:gateway-$GATEWAY_TAG-amd64" \ + "$ref:gateway-$GATEWAY_TAG-arm64" + + - name: Verify anonymous image access + run: | + docker logout ghcr.io + docker buildx imagetools inspect \ + "ghcr.io/contextforge-org/cf-integration-fixture:gateway-$GATEWAY_TAG" diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml index f06cd27..111a565 100644 --- a/.github/workflows/quality.yml +++ b/.github/workflows/quality.yml @@ -36,6 +36,9 @@ jobs: - name: Run full test suite run: cargo test --all-targets --locked + - name: Run FYRE campaign tests + run: python3 -m unittest discover -s benchmarks/fyre -p 'test_*.py' + - name: Verify standalone lazy runtime state shell: bash run: | @@ -61,4 +64,3 @@ jobs: - name: Validate GitHub Actions workflows run: go run github.com/rhysd/actionlint/cmd/actionlint@v1.7.12 - diff --git a/CHANGELOG.md b/CHANGELOG.md index d106a63..653f226 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,123 @@ and this project uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html) ## [Unreleased] +## [0.5.0] - 2026-09-16 + +### Added + +- Add `load fyre run|status|destroy` (with short aliases) and a packaged FYRE + Terraform campaign for matched vertical/horizontal Rust dataplane scaling. + The campaign pins provider and container versions, uses dedicated Locust and + Fast Time VMs, grows saturated helpers, captures host/container telemetry, + preserves raw reports before cleanup, and produces JSON, CSV, and a + Slack-ready comparison PNG. + +- Preflight FYRE product-group CPU, memory, standalone-VM Ubuntu root-disk, and + public-IP quota for the full scaling matrix before provisioning any benchmark + VM. Distinguish this standalone-VM limitation from the OCP cluster API's + configurable `base_disk_size`. + +- Add a repeatable low-memory vertical profile for 2 vCPU / 2 GB and + 4 vCPU / 4 GB dataplanes, with scenario multipliers derived from each + configuration's baseline resources. Allow profiles to start Locust and Fast + Time at independently validated helper sizes. + +- Add a reusable 2 vCPU / 2 GB FYRE profile for the complete eight-run built-in + dataplane versus external dataplane comparison. + +- Add `-w/--workers` to distribute load across local Locust processes, + `-i/--isolate-cpus` to split Docker CPUs between the target and load + generator, and `-m/--builtin-memory-limit` to tune the built-in gateway + without external environment setup. + +### Changed + +- Make bare `load fyre run` execute the CI-ready built-in-versus-Rust + comparison by default: the same modern `2026-07-28` client runs at 125, 250, + 500, and 1,000 users for one measured hour per lane on the same 4 vCPU / 4 GB + target VM. Produce combined JSON, CSV, and Slack-ready PNG artifacts with + matching-lane throughput ratios, and restart the full comparison with larger + helpers when telemetry shows helper saturation. + +### Fixed + +- Derive FYRE comparison report labels and target resources from the selected + profile instead of requiring and displaying a hard-coded 4 vCPU / 4 GB + target. + +- Quote inventory-derived backend URLs before composing every FYRE remote shell + command. + +- Pin the built-in comparison lane to the MCP SDK v2 gateway revision that + supports the same `2026-07-28` client as Rust, balance replicas across + distributed Locust workers, and keep benchmark services off FYRE public + interfaces. + +- Refine detected throughput plateaus to the configured concurrency boundary + before confirming capacity instead of confirming the highest doubled load. + +- Publish the dataplane's Redis-backed MCP Host and Origin policy before + startup, and isolate client-conformance scenarios from its per-user config + cache. + +- Discover the FYRE account's default or sole product group when no override is + configured, use API-compatible VM descriptions, and log in with the root SSH + account provisioned by FYRE's Ubuntu images. + +- Bootstrap FYRE hosts in parallel with pinned Ansible, installing Docker Engine + and Compose from Docker's Ubuntu repository when the base image lacks them. + +- Disable Ubuntu's automatic APT timers and services on FYRE benchmark hosts so + package upgrades cannot consume resources or restart services during a run. + +- Keep the FYRE dataplane and loopback JWKS helper in a stable shared network + namespace so either process can restart without breaking sidecar startup. + +- Omit an empty MCP allowed-origin environment value that prevented the Rust + dataplane from starting on FYRE. + +- Invoke the FYRE smoke script correctly through the Locust image's Python + entrypoint before beginning a capacity step. + +- Run FYRE smoke and Locust containers with access to the protected benchmark + bundle and root-owned report mounts on ephemeral load-generator VMs. + +- Use a valid Fast Time conversion timestamp and deploy the packaged Locust + workload to the FYRE load-generator VM. + +- Fully detach telemetry monitors from their SSH sessions so a benchmark phase + starts immediately instead of waiting for its own monitor to exit. + +- Accept Docker's empty-container telemetry form when evaluating helper and + dataplane pressure after a load phase. + +- Allow clean distributed-worker shutdown at the Locust time limit while still + stopping the coordinator immediately for nonzero or missing workers. + +- Record FYRE measurement boundaries from Locust's keyword-based spawning event + so ramp and warmup traffic remain excluded from reported statistics. + +- Propagate distributed worker failures to the Locust coordinator and reject + reports containing a hidden worker failure. + +- Raise the open-file limit for the load generator and external dataplane so + high-concurrency tests measure service capacity instead of Docker's low + default descriptor limit. + +- Remove the Locust client's 50-200 ms think time so load runs measure maximum + request throughput. + +- Refine below a provisional capacity when a confirmation repetition fails, + then confirm the lower zero-error boundary instead of aborting the remaining + FYRE comparison scenarios. + +- Keep each FYRE capacity step on one continuous Locust user population through + ramp, steady-state warmup, and measurement, resetting statistics only after + warmup completes. + +- Pull the pinned Locust and Redis images through the Google registry mirror and + retry transient container-pull failures with bounded backoff. + ## [0.4.0] - 2026-09-14 ### Added @@ -210,7 +327,9 @@ and this project uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html) - Added builtin and external dataplane routing through reusable Docker Compose overlays. -[Unreleased]: https://github.com/contextforge-org/contextforge-dev-tools/compare/v0.3.1...HEAD +[Unreleased]: https://github.com/contextforge-org/contextforge-dev-tools/compare/v0.5.0...HEAD +[0.5.0]: https://github.com/contextforge-org/contextforge-dev-tools/compare/v0.4.0...v0.5.0 +[0.4.0]: https://github.com/contextforge-org/contextforge-dev-tools/compare/v0.3.1...v0.4.0 [0.3.1]: https://github.com/contextforge-org/contextforge-dev-tools/compare/v0.3.0...v0.3.1 [0.3.0]: https://github.com/contextforge-org/contextforge-dev-tools/compare/v0.2.0...v0.3.0 [0.2.0]: https://github.com/contextforge-org/contextforge-dev-tools/compare/v0.1.0...v0.2.0 diff --git a/Cargo.lock b/Cargo.lock index 998e58e..72751d7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -201,7 +201,7 @@ dependencies = [ [[package]] name = "cf-integration" -version = "0.4.0" +version = "0.5.0" dependencies = [ "anyhow", "aws-lc-rs", diff --git a/Cargo.toml b/Cargo.toml index d077aa7..b84ab1e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "cf-integration" -version = "0.4.0" +version = "0.5.0" edition = "2024" rust-version = "1.97" license = "Apache-2.0" @@ -18,6 +18,9 @@ include = [ "/src/**", "/docker/**", "/scripts/locustfile_mcp.py", + "/benchmarks/fyre/**", + "!/benchmarks/fyre/**/__pycache__/**", + "!/benchmarks/fyre/**/*.pyc", "/scripts/live_protocol/sitecustomize.py", "/tests/conformance/baselines/**", "/README.md", diff --git a/README.md b/README.md index 8ea2058..f38ebcb 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ Every public command and option has a short form, shown in `--help`. | --- | --- | --- | | `stack` | `s` | `up` → `u`, `down` → `d`, `status` → `s`, `logs` → `l`, `config` → `c` | | `probe` | `p` | — | -| `load` | `l` | `run` → `r` | +| `load` | `l` | `run` → `r`, `fyre` → `f` (`run` → `r`, `status` → `s`, `destroy` → `d`) | | `live` | `v` | — | | `conformance` | `c` | `run` → `r`, `report` → `p` | | `debug` | `d` | `inspect` → `i`, `token` → `t` | @@ -179,12 +179,26 @@ cf-integration load run --lane external --client-era legacy --standalone \ # Include telemetry when diagnostic value matters more than benchmark purity cf-integration load run --lane external --client-era modern --standalone \ --observability --users 10 --spawn-rate 2 --run-time 2m + +# Spread a high-throughput run across eight local Locust workers +cf-integration load run --lane external --client-era modern --standalone \ + --workers 8 --isolate-cpus \ + --users 1000 --spawn-rate 100 --run-time 30s + +# Raise the built-in gateway limit for a high-concurrency comparison +cf-integration load run --lane builtin --client-era legacy \ + --builtin-memory-limit 16G --workers 8 \ + --isolate-cpus --users 1000 --spawn-rate 100 --run-time 30s ``` `--smoke` selects a short workload. Durations accept ordered positive `h`, `m`, and `s` groups such as `2m30s`. Defaults are `100` users, `10` users/s, and `5m`, overridable with `LOCUST_USERS`, `LOCUST_SPAWN_RATE`, and -`LOCUST_RUN_TIME`. Observability is opt-in for load tests to avoid skew. +`LOCUST_RUN_TIME`. `-w/--workers` starts that many local Locust worker +processes and defaults to one. `-m/--builtin-memory-limit` overrides the +built-in gateway container limit for that run. `-i/--isolate-cpus` splits all +Docker CPUs evenly between the selected target and Locust. Observability is +opt-in for load tests to avoid skew. `--client-era` accepts `legacy` or `modern` (default). The harness owns the Locust client: legacy uses initialization and the server's negotiated revision; @@ -195,8 +209,9 @@ Every load lane uses the Fast Time server with the same `CF_FAST_TIME_EXPECTED_I override and the same `echo` payload (`{"message":"cf-integration"}`). Pin that image to a digest when comparing lanes. The measured workload contains only `tools/call`; initialization/discovery and builtin tool-name discovery happen -once per user. Compare the `MCP tools/call` statistics to exclude setup traffic. -A missing echo tool fails the run instead of producing an empty benchmark. +once per user. Virtual users issue calls without client think time. Compare the +`MCP tools/call` statistics to exclude setup traffic. A missing echo tool fails +the run instead of producing an empty benchmark. There is no server-era selector: the backend must support the selected client era. `--standalone` runs Fast Time with the external dataplane and a harness @@ -204,6 +219,24 @@ routing snapshot in Redis, without the control plane. It discovers Fast Time's catalog directly; it never starts the conformance fixture or its proxy. Conformance, probes, and Inspector retain their protocol fixtures. +### FYRE scaling campaign + +Run the reproducible vertical and horizontal Rust dataplane comparison on FYRE: + +```bash +cf-integration load fyre run +cf-integration load fyre status --run-id scale-candidate +cf-integration load fyre destroy --run-id scale-candidate +``` + +The short forms are `cf-integration l f r`, `l f s`, and `l f d`; configuration +and run IDs use `-f` and `-i`. The packaged matrix, infrastructure lifecycle, +capacity-search rules, recovery behavior, and report layout are documented in +[`benchmarks/fyre/README.md`](benchmarks/fyre/README.md). FYRE credentials stay +in provider environment variables. All generated Terraform state, inventories, +raw reports, telemetry, manifests, and the Slack-ready PNG are kept under +`CF_INTEGRATION_DIR/fyre//`. + ## Live gateway checks Groups are `mcp`, `rbac`, `protocol`, and `all` (default): diff --git a/benchmarks/fyre/README.md b/benchmarks/fyre/README.md new file mode 100644 index 0000000..c14bc32 --- /dev/null +++ b/benchmarks/fyre/README.md @@ -0,0 +1,139 @@ +# FYRE built-in dataplane versus external dataplane benchmark + +The default FYRE workflow runs the same modern MCP client against the built-in +dataplane and the external dataplane on the same target VM. It +provisions three standalone Ubuntu 24.04 VMs and runs the two target stacks +sequentially so the target hardware is identical. + +| Role | Default allocation | Purpose | +| --- | ---: | --- | +| Locust | 4 vCPU / 16 GB | Three distributed `FastHttpUser` workers with zero wait | +| Target | 4 vCPU / 4 GB | Built-in dataplane or external dataplane, one lane at a time | +| Fast Time | 8 vCPU / 32 GB | Six nonfailure tools with explicit zero backend delay | + +The eight default measurements are built-in dataplane and external dataplane at 125, 250, 500, and +1,000 users. Every measurement ramps for 30 seconds, warms up for 30 seconds, +resets statistics, and records one hour. Both lanes send the same stateless +`2026-07-28` requests from the same Locust file. The built-in gateway owns any +session or backend protocol translation. + +The built-in dataplane target includes the Python gateway, PostgreSQL, and Redis. The +external dataplane target includes Rust, Redis, and the loopback JWKS helper. Both use +the same remote Fast Time VM and private FYRE network. + +The built-in image is pinned by digest and built from +`IBM/mcp-context-forge` commit +`33e2dd93a53a9cc2c5088b731822dfec4852fa2e` on the MCP SDK v2 branch. That +revision accepts the same `2026-07-28` stateless client used by the external dataplane lane. + +## Prerequisites + +- Terraform 1.8+, or `CF_TERRAFORM_BIN` pointing to a compatible binary. +- `python3`, `uv`, SSH, and SCP on the orchestration host. The CLI runs pinned + `ansible-core` through `uv` for host bootstrap. +- The SSH key pair configured in `scaling.yaml`. +- `FYRE_USERNAME` and `FYRE_API_KEY`. `FYRE_PRODUCT_GROUP_ID` and `FYRE_SITE` + are optional. + +Credentials are inherited by Terraform and are not written to manifests, +reports, or command arguments. Runtime benchmark tokens stay in mode-600 files +on the ephemeral VMs. + +FYRE's standalone VM API and the pinned provider allocate a 250 GB Ubuntu root +disk and expose no create-time root-disk setting. The default three-VM run +therefore needs 750 GB of FYRE disk quota. The CLI checks CPU, memory, disk, and +public-IP quota before it creates any benchmark VM. + +## Run the complete comparison + +The bare command is the CI entrypoint for the complete eight-run comparison: + +```bash +cf-integration load fyre run +``` + +Assigning a run ID makes artifact collection and recovery deterministic: + +```bash +cf-integration load fyre run --run-id builtin-rust-hourly +cf-integration l f r -i builtin-rust-hourly + +cf-integration load fyre status --run-id builtin-rust-hourly +cf-integration l f s -i builtin-rust-hourly + +cf-integration load fyre destroy --run-id builtin-rust-hourly +cf-integration l f d -i builtin-rust-hourly +``` + +A CI job only needs to expose the FYRE credentials, invoke the same command, +and upload `$CF_INTEGRATION_DIR/fyre//`. The command provisions the +VMs, bootstraps Docker, executes all eight measurements, downloads reports, +builds the comparison artifacts, and destroys only VMs owned by that run. + +Generated state lives under `$CF_INTEGRATION_DIR/fyre//`. Each run has +isolated Terraform state, and cleanup verifies its ownership record. Existing +manually created VMs are outside that state. + +The main outputs are: + +- `results/summary.csv` +- `results/summary.json` +- `results/slack-comparison.png` +- `results/comparison/-/` for Locust CSV, HTML, JSON, logs, and + host telemetry from each measured phase +- `manifest.json` for the exact images, allocation, workload, and inventory + +Artifacts are downloaded after every phase. On a request or worker error, the +campaign stops without advancing to the next load. On interruption or failure, +the CLI preserves downloaded artifacts, attempts a final remote recovery, +retries Terraform cleanup three times, and records `cleanup-failed` when a +manual `destroy` is still required. FYRE's 12-hour expiry is the final backstop +for the eight-hour measured campaign. + +## Fairness and helper headroom + +The client implementation, protocol revision, six-tool mix, request arguments, +zero delay, ramp, warmup, measurement window, request timeout, and target VM +allocation are identical between lanes. The report calculates `external +dataplane RPS ÷ built-in dataplane RPS` at each matching user count and includes request totals, errors, +and p50/p95/p99 latency. + +Telemetry covers CPU, per-core utilization, memory, swap and scheduling +pressure, socket counters, container health, worker exits, and virtualization +steal. Sustained helper pressure invalidates the partial comparison. The CLI +increases only the saturated helper through the configured sizes, archives the +partial run under `invalidated/`, and repeats all eight measurements so the +final rows use one helper allocation. Reaching 16 vCPU / 32 GB without helper +headroom leaves the campaign inconclusive. + +Benchmark service ports and the distributed Locust coordinator bind only to +private or loopback addresses. The public FYRE interfaces are used for SSH +orchestration only. + +## Optional capacity-search profile + +`vertical-low-memory.yaml` retains the earlier Rust-only capacity-search mode +for one 2 vCPU / 2 GB dataplane followed by one 4 vCPU / 4 GB dataplane: + +```bash +cf-integration load fyre run \ + --file benchmarks/fyre/vertical-low-memory.yaml \ + --run-id vertical-low-memory +``` + +That custom profile starts at 125 users, detects errors or a throughput plateau, +refines the boundary, and confirms the selected zero-error capacity three +times. It is separate from the default eight-run built-in-dataplane-versus-external-dataplane CI +comparison. + +Use the packaged two-core comparison profile to run the same eight measurements +on one 2 vCPU / 2 GB target VM: + +```bash +cf-integration load fyre run \ + --file benchmarks/fyre/comparison-2v2.yaml \ + --run-id builtin-external-2v2 +``` + +Comparison reports derive the target allocation from the selected profile; both +lanes always run sequentially on that same VM. diff --git a/benchmarks/fyre/ansible/bootstrap.yml b/benchmarks/fyre/ansible/bootstrap.yml new file mode 100644 index 0000000..91f6b0a --- /dev/null +++ b/benchmarks/fyre/ansible/bootstrap.yml @@ -0,0 +1,121 @@ +--- +- name: Wait for FYRE benchmark hosts + hosts: all + gather_facts: false + tasks: + - name: Wait for SSH + ansible.builtin.wait_for_connection: + delay: 5 + sleep: 5 + timeout: 600 + +- name: Bootstrap FYRE benchmark hosts + hosts: all + gather_facts: true + tasks: + - name: Disable automatic APT activity during benchmarks + ansible.builtin.copy: + dest: /etc/apt/apt.conf.d/99-contextforge-benchmark + mode: "0644" + content: | + APT::Periodic::Enable "0"; + APT::Periodic::Update-Package-Lists "0"; + APT::Periodic::Download-Upgradeable-Packages "0"; + APT::Periodic::AutocleanInterval "0"; + APT::Periodic::Unattended-Upgrade "0"; + + - name: Stop and mask automatic APT timers + ansible.builtin.systemd_service: + name: "{{ item }}" + state: stopped + enabled: false + masked: true + loop: + - apt-daily.timer + - apt-daily-upgrade.timer + + - name: Stop and mask automatic APT services + ansible.builtin.systemd_service: + name: "{{ item }}" + state: stopped + masked: true + loop: + - apt-daily.service + - apt-daily-upgrade.service + + - name: Install Docker repository prerequisites + ansible.builtin.apt: + name: + - ca-certificates + - curl + state: present + update_cache: true + + - name: Create the APT keyring directory + ansible.builtin.file: + path: /etc/apt/keyrings + state: directory + mode: "0755" + + - name: Install Docker's signing key + ansible.builtin.get_url: + url: https://download.docker.com/linux/ubuntu/gpg + dest: /etc/apt/keyrings/docker.asc + mode: "0644" + + - name: Configure Docker's Ubuntu repository + ansible.builtin.copy: + dest: /etc/apt/sources.list.d/docker.sources + mode: "0644" + content: | + Types: deb + URIs: https://download.docker.com/linux/ubuntu + Suites: {{ ansible_facts["distribution_release"] }} + Components: stable + Architectures: amd64 + Signed-By: /etc/apt/keyrings/docker.asc + + - name: Install Docker Engine, Compose, and telemetry tools + ansible.builtin.apt: + name: + - containerd.io + - docker-buildx-plugin + - docker-ce + - docker-ce-cli + - docker-compose-plugin + - iproute2 + state: present + update_cache: true + + - name: Enable Docker + ansible.builtin.service: + name: docker + state: started + enabled: true + + - name: Create benchmark directories + ansible.builtin.file: + path: "/root/cf-fyre/{{ item }}" + state: directory + mode: "0700" + loop: + - state/keys + - reports + - telemetry + + - name: Copy benchmark deployment assets + ansible.builtin.copy: + src: "{{ fyre_deploy_dir }}/" + dest: /root/cf-fyre/ + mode: preserve + + - name: Copy the Locust workload to the load generator + ansible.builtin.copy: + src: "{{ fyre_locustfile }}" + dest: /root/cf-fyre/locustfile_mcp.py + mode: "0644" + when: inventory_hostname == fyre_locust_hostname + + - name: Verify Docker and Compose + ansible.builtin.command: docker compose version + changed_when: false diff --git a/benchmarks/fyre/campaign.py b/benchmarks/fyre/campaign.py new file mode 100644 index 0000000..8bd9d07 --- /dev/null +++ b/benchmarks/fyre/campaign.py @@ -0,0 +1,1300 @@ +"""Bootstrap FYRE hosts and run a fixed comparison or capacity search.""" + +from __future__ import annotations + +import argparse +import csv +import json +import secrets +import shlex +import statistics +import subprocess +import tempfile +import time +from pathlib import Path + +HELPER_SATURATED = 42 +ANSIBLE_CORE_VERSION = "2.21.4" + + +def run( + arguments: list[str], + *, + check: bool = True, + capture: bool = False, + timeout: float | None = None, +) -> subprocess.CompletedProcess[str]: + return subprocess.run( + arguments, + check=check, + text=True, + timeout=timeout, + stdout=subprocess.PIPE if capture else None, + stderr=subprocess.PIPE if capture else None, + ) + + +class Remote: + def __init__(self, user: str, key: Path, known_hosts: Path): + self.user = user + self.options = [ + "-i", + str(key), + "-o", + "BatchMode=yes", + "-o", + "IdentitiesOnly=yes", + "-o", + f"UserKnownHostsFile={known_hosts}", + "-o", + "StrictHostKeyChecking=accept-new", + "-o", + "ConnectTimeout=10", + ] + + def ssh( + self, + host: str, + command: str, + *, + check: bool = True, + capture: bool = False, + timeout: float | None = None, + ): + return run( + ["ssh", *self.options, f"{self.user}@{host}", command], + check=check, + capture=capture, + timeout=timeout, + ) + + def copy_to(self, host: str, source: Path, destination: str) -> None: + destination = destination.removeprefix("~/") + run(["scp", *self.options, str(source), f"{self.user}@{host}:{destination}"]) + + def copy_from( + self, + host: str, + source: str, + destination: Path, + *, + recursive: bool = False, + check: bool = True, + ) -> None: + if recursive: + destination.mkdir(parents=True, exist_ok=True) + else: + destination.parent.mkdir(parents=True, exist_ok=True) + source = source.removeprefix("~/") + arguments = ["scp", *self.options] + if recursive: + arguments.append("-r") + run([*arguments, f"{self.user}@{host}:{source}", str(destination)], check=check) + + +def bootstrap_hosts( + config: dict, + inventory: dict, + deploy: Path, + playbook: Path, + known_hosts: Path, + output: Path, +) -> None: + hosts = [inventory["locust"], inventory["fast_time"], *inventory["dataplanes"]] + ansible_inventory = { + "all": { + "hosts": { + host["name"]: { + "ansible_host": host["public_ip"], + "ansible_user": config["infrastructure"]["ssh_user"], + "ansible_python_interpreter": "/usr/bin/python3", + } + for host in hosts + }, + "vars": { + "ansible_ssh_private_key_file": config["resolved_ssh_private_key"], + "ansible_ssh_common_args": " ".join( + [ + "-o BatchMode=yes", + "-o IdentitiesOnly=yes", + f"-o UserKnownHostsFile={known_hosts}", + "-o StrictHostKeyChecking=accept-new", + "-o ConnectTimeout=10", + ] + ), + }, + } + } + inventory_path = output / "ansible-inventory.json" + inventory_path.write_text( + json.dumps(ansible_inventory, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + run( + [ + "uv", + "tool", + "run", + "--from", + f"ansible-core=={ANSIBLE_CORE_VERSION}", + "ansible-playbook", + "--inventory", + str(inventory_path), + str(playbook), + "--extra-vars", + json.dumps( + { + "fyre_deploy_dir": str(deploy.resolve()), + "fyre_locustfile": str( + (deploy.parents[2] / "scripts/locustfile_mcp.py").resolve() + ), + "fyre_locust_hostname": inventory["locust"]["name"], + } + ), + ], + timeout=1_200, + ) + + +def write_remote_file( + remote: Remote, host: str, contents: str, destination: str, mode: int = 0o600 +) -> None: + destination = destination.removeprefix("~/") + with tempfile.NamedTemporaryFile("w", delete=False, encoding="utf-8") as stream: + stream.write(contents) + temporary = Path(stream.name) + try: + temporary.chmod(mode) + remote.copy_to(host, temporary, destination) + remote.ssh(host, f"chmod {mode:o} {shlex.quote(destination)}") + finally: + temporary.unlink(missing_ok=True) + + +def compose_up( + remote: Remote, host: str, compose: str, project: str | None = None +) -> None: + prefix = ( + "cd ~/cf-fyre && docker compose " + + (f"-p {shlex.quote(project)} " if project else "") + + "--env-file benchmark.env " + f"-f {shlex.quote(compose)}" + ) + pull = f"{prefix} pull" + retry_delays = (5, 15, 30) + for attempt in range(len(retry_delays) + 1): + result = remote.ssh(host, pull, check=False, timeout=900) + if result.returncode == 0: + break + if attempt == len(retry_delays): + result.check_returncode() + delay = retry_delays[attempt] + print( + f"container pull failed on {host}; retrying in {delay} seconds", + flush=True, + ) + time.sleep(delay) + remote.ssh( + host, + f"{prefix} up -d --wait", + timeout=900, + ) + + +def compose_down(remote: Remote, host: str, compose: str, project: str) -> None: + remote.ssh( + host, + "cd ~/cf-fyre && docker compose " + f"-p {shlex.quote(project)} --env-file benchmark.env " + f"-f {shlex.quote(compose)} down --volumes --remove-orphans", + check=False, + timeout=300, + ) + + +def prepare_hosts( + config: dict, + inventory: dict, + remote: Remote, + deploy: Path, + playbook: Path, + known_hosts: Path, + output: Path, +) -> tuple[str, list[str]]: + bootstrap_hosts(config, inventory, deploy, playbook, known_hosts, output) + + images = config["images"] + fast_env = "\n".join( + [ + f"FAST_TIME_IMAGE={images['fast_time']}", + f"FAST_TIME_BIND_IP={inventory['fast_time']['private_ip']}", + "", + ] + ) + write_remote_file( + remote, inventory["fast_time"]["public_ip"], fast_env, "~/cf-fyre/benchmark.env" + ) + compose_up(remote, inventory["fast_time"]["public_ip"], "fast-time.compose.yaml") + + first = inventory["dataplanes"][0] + for index, target in enumerate(inventory["dataplanes"]): + allowed = ",".join( + [ + f"{target['private_ip']}:4445", + f"{target['public_ip']}:4445", + "127.0.0.1:4445", + "localhost:4445", + ] + ) + target_env = "\n".join( + [ + f"DATAPLANE_IMAGE={images['dataplane']}", + f"HELPERS_IMAGE={images['helpers']}", + f"REDIS_IMAGE={images['redis']}", + f"TARGET_BIND_IP={target['private_ip']}", + f"DATAPLANE_ALLOWED_HOSTS={allowed}", + f"CONFIG_CACHE_SECONDS={config['workload']['config_cache_seconds']}", + "", + ] + ) + write_remote_file( + remote, target["public_ip"], target_env, "~/cf-fyre/benchmark.env" + ) + if index > 0: + with tempfile.TemporaryDirectory() as temporary: + key = Path(temporary) / "jwt.key" + remote.copy_from( + first["public_ip"], "~/cf-fyre/state/keys/jwt.key", key + ) + remote.copy_to(target["public_ip"], key, "~/cf-fyre/state/keys/jwt.key") + remote.ssh( + target["public_ip"], "chmod 600 ~/cf-fyre/state/keys/jwt.key" + ) + compose_up(remote, target["public_ip"], "dataplane.compose.yaml") + if index == 0: + # The first auth container creates the campaign key; subsequent replicas receive it. + remote.ssh( + first["public_ip"], + "test -s ~/cf-fyre/state/keys/jwt.key && chown root:root ~/cf-fyre/state/keys/jwt.key && chmod 600 ~/cf-fyre/state/keys/jwt.key", + ) + + token = remote.ssh( + first["public_ip"], + "cd ~/cf-fyre && docker compose --env-file benchmark.env -f dataplane.compose.yaml run --rm --no-deps config_writer token fyre-benchmark fyre-user", + capture=True, + ).stdout.strip() + if not token or "\n" in token: + raise RuntimeError("config helper did not return one bearer token") + token_file = output / ".token" + token_file.write_text(token, encoding="utf-8") + token_file.chmod(0o600) + try: + backend_url = f"http://{inventory['fast_time']['private_ip']}:9080/mcp" + for target in inventory["dataplanes"]: + remote.copy_to(target["public_ip"], token_file, "~/cf-fyre/state/token") + remote.ssh(target["public_ip"], "chmod 600 ~/cf-fyre/state/token") + remote.ssh( + target["public_ip"], + 'cd ~/cf-fyre && export MCP_CONFORMANCE_TOKEN="$(cat state/token)" && ' + "docker compose --env-file benchmark.env -f dataplane.compose.yaml " + "run --rm --no-deps -e MCP_CONFORMANCE_TOKEN config_writer " + f"fixture fyre-fast-time {shlex.quote(backend_url)} " + f"{shlex.quote(config['workload']['protocol_version'])}", + timeout=120, + ) + locust_env = "\n".join( + [ + f"MCPGATEWAY_BEARER_TOKEN={token}", + "MCP_PROTOCOL_VERSION=2026-07-28", + "MCP_STACK_MODE=dataplane", + "MCP_SERVER_ID=fyre-fast-time", + "MCP_DIRECT_DATAPLANE=true", + "MCP_SKIP_TOOL_LIST=true", + "MCP_EXPLICIT_ZERO_DELAY=true", + "MCP_FYRE_WORKLOAD=true", + "MCP_TOOL_NAMES=convert_time,echo,get_stats,get_system_time,schema_success,verify-protocol", + "LOCUST_REQUEST_TIMEOUT_SECONDS=30", + "MCP_BASE_URLS=" + + ",".join( + f"http://{target['private_ip']}:4445" + for target in inventory["dataplanes"] + ), + "", + ] + ) + write_remote_file( + remote, + inventory["locust"]["public_ip"], + locust_env, + "~/cf-fyre/benchmark.secret.env", + ) + direct_env = "\n".join( + [ + f"MCPGATEWAY_BEARER_TOKEN={token}", + "MCP_PROTOCOL_VERSION=2026-07-28", + "MCP_STACK_MODE=controlplane", + "MCP_FYRE_WORKLOAD=true", + "MCP_SKIP_TOOL_LIST=true", + "MCP_EXPLICIT_ZERO_DELAY=true", + "MCP_TOOL_NAMES=convert_time,echo,get_stats,get_system_time,schema_success,verify-protocol", + "LOCUST_REQUEST_TIMEOUT_SECONDS=30", + f"MCP_BASE_URLS=http://{inventory['fast_time']['private_ip']}:9080", + "", + ] + ) + write_remote_file( + remote, + inventory["locust"]["public_ip"], + direct_env, + "~/cf-fyre/direct.secret.env", + ) + remote.copy_to( + inventory["locust"]["public_ip"], token_file, "~/cf-fyre/state/token" + ) + remote.ssh(inventory["locust"]["public_ip"], "chmod 600 ~/cf-fyre/state/token") + finally: + token_file.unlink(missing_ok=True) + urls = [ + f"http://{target['private_ip']}:4445/contextforge-rs/servers/fyre-fast-time/mcp" + for target in inventory["dataplanes"] + ] + return token, urls + + +def start_monitor(remote: Remote, host: str, role: str, name: str) -> int: + command = ( + "nohup python3 cf-fyre/monitor.py" + f" --role {shlex.quote(role)}" + f" --output cf-fyre/telemetry/{shlex.quote(name)}.jsonl" + " cf-fyre/telemetry/{shlex.quote(name)}.log 2>&1 & echo $!" + ) + return int(remote.ssh(host, command, capture=True).stdout.strip()) + + +def stop_monitor(remote: Remote, host: str, pid: int) -> None: + remote.ssh( + host, + f"kill -TERM {pid} 2>/dev/null || true; wait {pid} 2>/dev/null || true", + check=False, + ) + + +def smoke( + remote: Remote, + locust: dict, + urls: list[str], + locust_image: str, + tool_names: list[str] | None = None, +) -> None: + command = " ".join( + [ + "cd ~/cf-fyre && docker run --rm --user 0:0 --network host --entrypoint python", + "-v $HOME/cf-fyre:/work -w /work", + shlex.quote(locust_image), + "smoke.py --urls", + shlex.quote(",".join(urls)), + "--token-file state/token", + "--tool-names", + shlex.quote(",".join(tool_names) if tool_names else ",".join(configured_tools())), + ] + ) + remote.ssh(locust["public_ip"], command, timeout=120) + + +def configured_tools() -> list[str]: + return [ + "convert_time", + "echo", + "get_stats", + "get_system_time", + "schema_success", + "verify-protocol", + ] + + +def read_stats(path: Path, use_aggregate: bool = False) -> dict: + with path.open(newline="", encoding="utf-8") as stream: + rows = list(csv.DictReader(stream)) + tool_rows = [ + row for row in rows if row.get("Name", "").startswith("MCP tools/call") + ] + if not tool_rows: + raise RuntimeError(f"Locust report {path} contains no measured tool traffic") + aggregate = next((row for row in rows if row.get("Name") == "Aggregated"), None) + selected = [aggregate] if use_aggregate and aggregate is not None else tool_rows + failures = sum(int(float(row.get("Failure Count") or 0)) for row in selected) + requests = sum(int(float(row.get("Request Count") or 0)) for row in selected) + rps = sum(float(row.get("Requests/s") or 0) for row in selected) + per_replica = {row["Name"]: float(row.get("Requests/s") or 0) for row in tool_rows} + + def weighted(column: str) -> float: + return ( + sum( + float(row.get(column) or 0) * int(float(row.get("Request Count") or 0)) + for row in selected + ) + / requests + if requests + else 0.0 + ) + + return { + "requests": requests, + "failures": failures, + "rps": rps, + "p50_ms": weighted("50%"), + "p95_ms": weighted("95%"), + "p99_ms": weighted("99%"), + "per_replica_rps": per_replica, + } + + +def kernel_counter(text: str, name: str) -> int: + lines = text.splitlines() + for header, values in zip(lines[0::2], lines[1::2]): + header_fields = header.split() + value_fields = values.split() + if not header_fields or not value_fields or header_fields[0] != value_fields[0]: + continue + try: + return int(value_fields[header_fields.index(name)]) + except (ValueError, IndexError): + continue + return 0 + + +def docker_pressure(text: str) -> bool: + for line in text.splitlines(): + try: + parsed = json.loads(line) + except ValueError: + continue + states = parsed if isinstance(parsed, list) else [parsed] + for state in states: + if not isinstance(state, dict): + continue + health = state.get("Health") or {} + if ( + state.get("OOMKilled") is True + or state.get("Status") == "dead" + or (state.get("Status") == "exited" and state.get("ExitCode") != 0) + or health.get("Status") == "unhealthy" + ): + return True + return False + + +def pressure(path: Path, after: float | None = None) -> dict[str, float | bool]: + samples = [] + for line in path.read_text(encoding="utf-8").splitlines(): + item = json.loads(line) + if item.get("kind") == "sample" and ( + after is None or item.get("time", 0) >= after + ): + samples.append(item) + busy = [ + item["cpu"]["cpu"]["busy_percent"] + for item in samples + if "cpu" in item.get("cpu", {}) + ] + memory = [item["memory"]["used_percent"] for item in samples] + core_names = { + key for item in samples for key in item.get("cpu", {}) if key != "cpu" + } + core_means = [ + statistics.fmean( + item["cpu"][key]["busy_percent"] + for item in samples + if key in item.get("cpu", {}) + ) + for key in core_names + ] + steal = [ + item["cpu"]["cpu"]["steal_percent"] + for item in samples + if "cpu" in item.get("cpu", {}) + ] + network_counters = [ + sum( + kernel_counter(item.get("netstat", ""), counter) + for counter in ("ListenOverflows", "ListenDrops", "TCPBacklogDrop") + ) + for item in samples + ] + docker_unhealthy = any( + docker_pressure(item.get("docker_state", "")) for item in samples + ) + return { + "mean_cpu_percent": statistics.fmean(busy) if busy else 0.0, + "max_memory_percent": max(memory, default=0.0), + "max_mean_core_percent": max(core_means, default=0.0), + "mean_steal_percent": statistics.fmean(steal) if steal else 0.0, + "worker_or_network_pressure": docker_unhealthy + or (len(network_counters) > 1 and network_counters[-1] > network_counters[0]), + } + + +def one_phase( + remote: Remote, + config: dict, + inventory: dict, + urls: list[str], + output: Path, + users: int, + seconds: int, + label: str, + env_file: str = "benchmark.secret.env", + target_role_prefix: str = "dataplane", +) -> dict: + locust = inventory["locust"] + workers = max(2, int(config["active_helper"]["locust_cpu"]) - 1) + spawn_rate = max(1.0, users / config["workload"]["ramp_seconds"]) + total_seconds = ( + seconds + + config["workload"]["ramp_seconds"] + + config["workload"]["warmup_seconds"] + ) + remote_output = f"reports/{label}" + monitors: list[tuple[str, int]] = [] + monitor_hosts = [ + (locust, "locust"), + (inventory["fast_time"], "fast-time"), + *[ + (target, f"{target_role_prefix}-{index + 1}") + for index, target in enumerate(inventory["dataplanes"]) + ], + ] + for host, role in monitor_hosts: + monitors.append( + (host["public_ip"], start_monitor(remote, host["public_ip"], role, label)) + ) + try: + command = " ".join( + [ + "cd ~/cf-fyre && python3 run_locust.py", + "--image", + shlex.quote(config["images"]["locust"]), + "--users", + str(users), + "--spawn-rate", + str(spawn_rate), + "--seconds", + str(total_seconds), + "--workers", + str(workers), + "--output", + shlex.quote(remote_output), + "--env-file", + shlex.quote(env_file), + "--reset-stats", + "--measurement-seconds", + str(seconds), + "--warmup-seconds", + str(config["workload"]["warmup_seconds"]), + ] + ) + result = remote.ssh( + locust["public_ip"], command, check=False, timeout=total_seconds + 180 + ) + finally: + for host, pid in monitors: + stop_monitor(remote, host, pid) + local = output / label + local.mkdir(parents=True, exist_ok=True) + remote.copy_from( + locust["public_ip"], + f"~/cf-fyre/{remote_output}/.", + local, + recursive=True, + check=False, + ) + pressures = {} + marker = local / "measurement-start.txt" + measurement_start = ( + float(marker.read_text(encoding="utf-8").strip()) + if marker.is_file() + else None + ) + for host, role in monitor_hosts: + path = local / f"{role}.jsonl" + remote.copy_from( + host["public_ip"], f"~/cf-fyre/telemetry/{label}.jsonl", path, check=False + ) + if path.exists(): + pressures[role] = pressure(path, after=measurement_start) + if measurement_start is None: + return { + "passed": False, + "reason": "Locust did not record the measurement-window start", + "pressure": pressures, + } + if result.returncode != 0: + return { + "passed": False, + "reason": f"Locust exited {result.returncode}", + "pressure": pressures, + } + stats = read_stats(local / "locust_stats.csv", use_aggregate=True) + stats.update( + {"passed": stats["failures"] == 0, "pressure": pressures, "users": users} + ) + return stats + + +def helper_saturation(config: dict, result: dict) -> str | None: + workload = config["workload"] + for role in ("locust", "fast-time"): + values = result.get("pressure", {}).get(role, {}) + if values.get("mean_cpu_percent", 0) > workload["helper_cpu_percent"]: + return role + if values.get("max_memory_percent", 0) > workload["helper_memory_percent"]: + return role + if values.get("max_mean_core_percent", 0) > workload["worker_core_percent"]: + return role + if values.get("worker_or_network_pressure", False): + return role + return None + + +def measured_step( + remote: Remote, + config: dict, + inventory: dict, + urls: list[str], + output: Path, + users: int, + name: str, +) -> dict: + smoke(remote, inventory["locust"], urls, config["images"]["locust"]) + result = one_phase( + remote, + config, + inventory, + urls, + output, + users, + config["workload"]["measure_seconds"], + name, + ) + saturated = helper_saturation(config, result) + if saturated: + (output / "helper-request.json").write_text( + json.dumps({"role": saturated}, indent=2) + "\n", encoding="utf-8" + ) + raise SystemExit(HELPER_SATURATED) + return result + + +def capacity_search( + remote: Remote, config: dict, inventory: dict, urls: list[str], output: Path +) -> dict: + workload = config["workload"] + started = time.monotonic() + passing: list[dict] = [] + failing: dict | None = None + plateau: tuple[dict, dict] | None = None + improvements: list[float] = [] + users = workload["first_users"] + step = 0 + while ( + users <= workload["maximum_users"] + and time.monotonic() - started < workload["maximum_campaign_seconds"] + ): + step += 1 + result = measured_step( + remote, config, inventory, urls, output, users, f"search-{step}-{users}" + ) + if not result.get("passed"): + failing = {"users": users, **result} + break + if passing: + improvements.append(100.0 * (result["rps"] / passing[-1]["rps"] - 1.0)) + passing.append(result) + if len(improvements) >= 2 and all( + value < workload["plateau_improvement_percent"] + for value in improvements[-2:] + ): + plateau = (passing[-3], passing[-2]) + break + if users == workload["maximum_users"]: + break + users = min(workload["maximum_users"], users * 2) + + if not passing: + return { + "status": "failed", + "reason": "no zero-error concurrency passed", + "failing": failing, + } + + def refine_below(high: int, label: str) -> dict | None: + nonlocal failing + eligible = [item for item in passing if item["users"] < high] + if not eligible: + return None + low_result = max(eligible, key=lambda item: item["users"]) + low = low_result["users"] + while (high - low) / high > workload["boundary_percent"] / 100.0: + users = (low + high) // 2 + result = measured_step( + remote, config, inventory, urls, output, users, f"{label}-{users}" + ) + if result.get("passed"): + passing.append(result) + low = users + low_result = result + else: + failing = {"users": users, **result} + high = users + return low_result + + def refine_plateau(low_result: dict, high_result: dict) -> tuple[dict, dict]: + nonlocal failing + low = low_result["users"] + high = high_result["users"] + while (high - low) / high > workload["boundary_percent"] / 100.0: + refined_users = (low + high) // 2 + result = measured_step( + remote, + config, + inventory, + urls, + output, + refined_users, + f"plateau-refine-{refined_users}", + ) + if not result.get("passed"): + failing = {"users": refined_users, **result} + refined = refine_below(refined_users, "plateau-failure-refine") + if refined is None: + return low_result, { + "below": low_result, + "at_or_above": failing, + } + return refined, { + "below": refined, + "at_or_above": failing, + } + passing.append(result) + improvement = 100.0 * (result["rps"] / low_result["rps"] - 1.0) + if improvement >= workload["plateau_improvement_percent"]: + low = refined_users + low_result = result + else: + high = refined_users + high_result = result + return high_result, { + "below": low_result, + "at_or_above": high_result, + "width_percent": 100.0 * (high - low) / high, + } + + plateau_boundary = None + if failing: + candidate = refine_below(failing["users"], "refine") + if candidate is None: + return { + "status": "failed", + "reason": "no zero-error concurrency passed below the failing bound", + "failing": failing, + } + elif plateau: + candidate, plateau_boundary = refine_plateau(*plateau) + else: + candidate = max(passing, key=lambda item: item["users"]) + + confirmation_failures = [] + while True: + confirmations = [] + failure = None + for repetition in range(workload["repetitions"]): + result = measured_step( + remote, + config, + inventory, + urls, + output, + candidate["users"], + f"confirm-{repetition + 1}-{candidate['users']}", + ) + if not result.get("passed"): + failure = result + break + confirmations.append(result) + if failure is None: + break + failing = {"users": candidate["users"], **failure} + confirmation_failures.append( + {"candidate": candidate, "confirmations": confirmations, "failure": failure} + ) + refined = refine_below( + candidate["users"], f"confirm-refine-{len(confirmation_failures)}" + ) + if refined is None: + return { + "status": "failed-confirmation", + "candidate": candidate, + "confirmations": confirmations, + "confirmation_failures": confirmation_failures, + "failure": failure, + } + candidate = refined + direct_url = f"http://{inventory['fast_time']['private_ip']}:9080/mcp" + smoke(remote, inventory["locust"], [direct_url], config["images"]["locust"]) + calibration = one_phase( + remote, + config, + inventory, + [direct_url], + output, + candidate["users"], + workload["measure_seconds"], + "calibration", + "direct.secret.env", + ) + saturated = helper_saturation(config, calibration) + if saturated: + (output / "helper-request.json").write_text( + json.dumps({"role": saturated}, indent=2) + "\n", encoding="utf-8" + ) + raise SystemExit(HELPER_SATURATED) + if ( + not calibration.get("passed") + or calibration.get("rps", 0) < min(item["rps"] for item in confirmations) * 1.05 + ): + return { + "status": "inconclusive", + "reason": "direct Fast Time calibration did not demonstrate five percent upstream headroom", + "calibration": calibration, + } + rps_values = [result["rps"] for result in confirmations] + imbalances = [] + for result in confirmations: + replicas = list(result["per_replica_rps"].values()) + mean = statistics.fmean(replicas) if replicas else 0.0 + imbalances.append( + 100.0 * (max(replicas) - min(replicas)) / mean + if mean and len(replicas) > 1 + else 0.0 + ) + best = min(confirmations, key=lambda item: item["rps"]) + return { + "status": "confirmed", + "users": candidate["users"], + "search": passing, + "failing": failing, + "confirmations": confirmations, + "confirmation_failures": confirmation_failures, + "plateau_boundary": plateau_boundary, + "rps": statistics.fmean(rps_values), + "rps_min": min(rps_values), + "rps_max": max(rps_values), + "rps_cv_percent": 100.0 + * statistics.pstdev(rps_values) + / statistics.fmean(rps_values) + if len(rps_values) > 1 + else 0.0, + "replica_imbalance_percent": statistics.fmean(imbalances), + "p50_ms": statistics.fmean(item["p50_ms"] for item in confirmations), + "p95_ms": statistics.fmean(item["p95_ms"] for item in confirmations), + "p99_ms": statistics.fmean(item["p99_ms"] for item in confirmations), + "lower_bound": candidate["users"] == workload["maximum_users"] + and failing is None, + "conservative_confirmation": best, + "direct_backend_calibration": calibration, + } + + +def write_lane_environment( + remote: Remote, + locust: dict, + token: str, + protocol_version: str, + stack_mode: str, + base_url: str, + tool_names: list[str], + destination: str, +) -> None: + values = [ + f"MCPGATEWAY_BEARER_TOKEN={token}", + f"MCP_PROTOCOL_VERSION={protocol_version}", + f"MCP_STACK_MODE={stack_mode}", + "MCP_SERVER_ID=fyre-fast-time", + "MCP_DIRECT_DATAPLANE=true" if stack_mode == "dataplane" else "MCP_DIRECT_DATAPLANE=false", + "MCP_SKIP_TOOL_LIST=true", + "MCP_EXPLICIT_ZERO_DELAY=true", + "MCP_FYRE_WORKLOAD=true", + f"MCP_TOOL_NAMES={','.join(tool_names)}", + "LOCUST_REQUEST_TIMEOUT_SECONDS=30", + f"MCP_BASE_URLS={base_url}", + "", + ] + write_remote_file( + remote, + locust["public_ip"], + "\n".join(values), + f"~/cf-fyre/{destination}", + ) + write_remote_file( + remote, + locust["public_ip"], + token, + "~/cf-fyre/state/token", + ) + + +def prepare_comparison_shared( + config: dict, + inventory: dict, + remote: Remote, + deploy: Path, + playbook: Path, + known_hosts: Path, + output: Path, +) -> None: + bootstrap_hosts(config, inventory, deploy, playbook, known_hosts, output) + write_remote_file( + remote, + inventory["fast_time"]["public_ip"], + "\n".join( + [ + f"FAST_TIME_IMAGE={config['images']['fast_time']}", + f"FAST_TIME_BIND_IP={inventory['fast_time']['private_ip']}", + "", + ] + ), + "~/cf-fyre/benchmark.env", + ) + compose_up(remote, inventory["fast_time"]["public_ip"], "fast-time.compose.yaml") + + +def reset_comparison_target(remote: Remote, target: dict) -> None: + for compose, project in ( + ("dataplane.compose.yaml", "cf-fyre-rust"), + ("builtin.compose.yaml", "cf-fyre-builtin"), + ): + compose_down(remote, target["public_ip"], compose, project) + remote.ssh( + target["public_ip"], + "rm -rf ~/cf-fyre/state/keys ~/cf-fyre/state/token && mkdir -p ~/cf-fyre/state/keys", + ) + + +def prepare_rust_comparison( + config: dict, inventory: dict, remote: Remote +) -> tuple[list[str], list[str], str]: + target = inventory["dataplanes"][0] + reset_comparison_target(remote, target) + allowed = ",".join( + [ + f"{target['private_ip']}:4445", + f"{target['public_ip']}:4445", + "127.0.0.1:4445", + "localhost:4445", + ] + ) + images = config["images"] + write_remote_file( + remote, + target["public_ip"], + "\n".join( + [ + f"DATAPLANE_IMAGE={images['dataplane']}", + f"HELPERS_IMAGE={images['helpers']}", + f"REDIS_IMAGE={images['redis']}", + f"TARGET_BIND_IP={target['private_ip']}", + f"DATAPLANE_ALLOWED_HOSTS={allowed}", + f"CONFIG_CACHE_SECONDS={config['workload']['config_cache_seconds']}", + "", + ] + ), + "~/cf-fyre/benchmark.env", + ) + compose_up( + remote, + target["public_ip"], + "dataplane.compose.yaml", + "cf-fyre-rust", + ) + prefix = ( + "cd ~/cf-fyre && docker compose -p cf-fyre-rust " + "--env-file benchmark.env -f dataplane.compose.yaml run --rm --no-deps config_writer" + ) + token = remote.ssh( + target["public_ip"], + f"{prefix} token fyre-benchmark fyre-user", + capture=True, + timeout=120, + ).stdout.strip() + if not token or "\n" in token: + raise RuntimeError("config helper did not return one bearer token") + write_remote_file( + remote, + target["public_ip"], + token, + "~/cf-fyre/state/token", + ) + backend_url = f"http://{inventory['fast_time']['private_ip']}:9080/mcp" + remote.ssh( + target["public_ip"], + 'cd ~/cf-fyre && export MCP_CONFORMANCE_TOKEN="$(cat state/token)" && ' + f"docker compose -p cf-fyre-rust --env-file benchmark.env " + f"-f dataplane.compose.yaml run --rm --no-deps -e MCP_CONFORMANCE_TOKEN " + f"config_writer fixture fyre-fast-time " + f"{shlex.quote(backend_url)} " + f"{config['workload']['protocol_version']}", + timeout=120, + ) + tools = list(config["workload"]["tools"]) + write_lane_environment( + remote, + inventory["locust"], + token, + config["workload"]["protocol_version"], + "dataplane", + f"http://{target['private_ip']}:4445", + tools, + "rust.secret.env", + ) + return ( + [f"http://{target['private_ip']}:4445/contextforge-rs/servers/fyre-fast-time/mcp"], + tools, + "rust.secret.env", + ) + + +def prepare_builtin_comparison( + config: dict, inventory: dict, remote: Remote +) -> tuple[list[str], list[str], str]: + target = inventory["dataplanes"][0] + reset_comparison_target(remote, target) + images = config["images"] + password = secrets.token_hex(24) + target_env = "\n".join( + [ + f"CONTROLPLANE_IMAGE={images['controlplane']}", + f"HELPERS_IMAGE={images['helpers']}", + f"POSTGRES_IMAGE={images['postgres']}", + f"REDIS_IMAGE={images['redis']}", + f"TARGET_BIND_IP={target['private_ip']}", + f"POSTGRES_PASSWORD={secrets.token_hex(24)}", + f"JWT_SECRET_KEY={secrets.token_hex(32)}", + f"AUTH_ENCRYPTION_SECRET={secrets.token_hex(32)}", + f"DEFAULT_USER_PASSWORD={password}", + f"PLATFORM_ADMIN_PASSWORD={password}", + "", + ] + ) + write_remote_file( + remote, target["public_ip"], target_env, "~/cf-fyre/benchmark.env" + ) + compose_up( + remote, + target["public_ip"], + "builtin.compose.yaml", + "cf-fyre-builtin", + ) + backend_url = f"http://{inventory['fast_time']['private_ip']}:9080/mcp" + command = ( + "cd ~/cf-fyre && docker compose -p cf-fyre-builtin " + "--env-file benchmark.env -f builtin.compose.yaml run --rm --no-deps admin " + f"--backend {shlex.quote(backend_url)}" + ) + output = remote.ssh( + target["public_ip"], command, capture=True, timeout=300 + ).stdout.splitlines() + if not output: + raise RuntimeError("built-in registration returned no result") + registration = json.loads(output[-1]) + tools = registration["tool_names"] + if len(tools) != 6: + raise RuntimeError("built-in registration did not select six benchmark tools") + write_lane_environment( + remote, + inventory["locust"], + registration["token"], + config["workload"]["protocol_version"], + "controlplane", + f"http://{target['private_ip']}:4444", + tools, + "builtin.secret.env", + ) + return ( + [f"http://{target['private_ip']}:4444/mcp"], + tools, + "builtin.secret.env", + ) + + +def fixed_comparison( + remote: Remote, config: dict, inventory: dict, output: Path +) -> dict: + result = { + "status": "running", + "scenario": config["scenarios"][0], + "inventory": inventory, + "protocol_version": config["workload"]["protocol_version"], + "user_levels": config["workload"]["user_levels"], + "runs": {"rust": [], "builtin": []}, + } + result_path = output / "result.json" + + def save() -> None: + result_path.write_text( + json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + + save() + target = inventory["dataplanes"][0] + try: + for lane, prepare in ( + ("rust", prepare_rust_comparison), + ("builtin", prepare_builtin_comparison), + ): + urls, tools, env_file = prepare(config, inventory, remote) + for users in config["workload"]["user_levels"]: + smoke( + remote, + inventory["locust"], + urls, + config["images"]["locust"], + tools, + ) + phase = one_phase( + remote, + config, + inventory, + urls, + output, + users, + config["workload"]["measure_seconds"], + f"{lane}-{users}", + env_file, + "target", + ) + phase["lane"] = lane + result["runs"][lane].append(phase) + save() + saturated = helper_saturation(config, phase) + if saturated: + result.update( + { + "status": "inconclusive", + "reason": f"{saturated} helper saturated at {users} users", + } + ) + save() + (output / "helper-request.json").write_text( + json.dumps({"role": saturated}, indent=2) + "\n", + encoding="utf-8", + ) + raise SystemExit(HELPER_SATURATED) + if not phase.get("passed"): + result.update( + { + "status": "failed", + "reason": phase.get("reason", f"{lane} failed at {users} users"), + } + ) + save() + return result + reset_comparison_target(remote, target) + result["status"] = "confirmed" + save() + return result + except BaseException as error: + if isinstance(error, SystemExit) and error.code == HELPER_SATURATED: + raise + result.update({"status": "failed", "reason": str(error)}) + save() + raise + finally: + reset_comparison_target(remote, target) + + +def collect_recovery(remote: Remote, inventory: dict, output: Path) -> None: + recovery = output / "recovery" + recovery.mkdir(parents=True, exist_ok=True) + for host in [inventory["locust"], inventory["fast_time"], *inventory["dataplanes"]]: + remote.ssh( + host["public_ip"], + "pkill -TERM -f 'python3 monitor.py' 2>/dev/null || true; docker ps --filter name=cf-fyre --format '{{.ID}}' | xargs -r docker rm -f >/dev/null 2>&1 || true", + check=False, + ) + destination = recovery / host["name"] + destination.mkdir(exist_ok=True) + remote.copy_from( + host["public_ip"], + "cf-fyre/reports/.", + destination / "reports", + recursive=True, + check=False, + ) + remote.copy_from( + host["public_ip"], + "cf-fyre/telemetry/.", + destination / "telemetry", + recursive=True, + check=False, + ) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--config", required=True) + parser.add_argument("--inventory", required=True) + parser.add_argument("--scenario", required=True) + parser.add_argument("--deploy", required=True) + parser.add_argument("--ansible", required=True) + parser.add_argument("--output", required=True) + parser.add_argument("--collect-only", action="store_true") + args = parser.parse_args() + config = json.loads(Path(args.config).read_text(encoding="utf-8")) + inventory = json.loads(Path(args.inventory).read_text(encoding="utf-8")) + scenario = next(item for item in config["scenarios"] if item["id"] == args.scenario) + output = Path(args.output) + output.mkdir(parents=True, exist_ok=True) + known_hosts = output.parent.parent / "known_hosts" + known_hosts.touch(exist_ok=True) + remote = Remote( + config["infrastructure"]["ssh_user"], + Path(config["resolved_ssh_private_key"]), + known_hosts, + ) + if args.collect_only: + collect_recovery(remote, inventory, output) + return + if config.get("benchmark_kind", "scaling") == "comparison": + prepare_comparison_shared( + config, + inventory, + remote, + Path(args.deploy), + Path(args.ansible), + known_hosts, + output, + ) + result = fixed_comparison(remote, config, inventory, output) + else: + _, urls = prepare_hosts( + config, + inventory, + remote, + Path(args.deploy), + Path(args.ansible), + known_hosts, + output, + ) + result = capacity_search(remote, config, inventory, urls, output) + result["scenario"] = scenario + result["inventory"] = inventory + (output / "result.json").write_text( + json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + if result["status"] != "confirmed": + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/fyre/comparison-2v2.yaml b/benchmarks/fyre/comparison-2v2.yaml new file mode 100644 index 0000000..cbd4f4e --- /dev/null +++ b/benchmarks/fyre/comparison-2v2.yaml @@ -0,0 +1,48 @@ +schema_version: 1 +benchmark_kind: comparison +infrastructure: + os: Ubuntu 24.04 + ssh_user: root + ssh_private_key: ~/.ssh/id_ed25519 + ssh_public_key: ~/.ssh/id_ed25519.pub + expiry_hours: 12 + helper_sizes: + - { cpu: 4, memory_gb: 16 } + - { cpu: 8, memory_gb: 32 } + - { cpu: 16, memory_gb: 32 } + initial_helpers: + locust: { cpu: 4, memory_gb: 16 } + fast_time: { cpu: 8, memory_gb: 32 } +images: + dataplane: ghcr.io/contextforge-org/contextforge-data-plane@sha256:0b3026b21659ce6c494142aee2c5866b44122232e73fcf21102da1f210692a96 + controlplane: ghcr.io/contextforge-org/cf-integration-fixture@sha256:5b206e6f863cea9f8cabea6451392428fbe67b16bdf10fefd1f7ac8032d95b52 + fast_time: ghcr.io/ibm/cfex-mcp-fast-time-server@sha256:110e1826f5d763e5afadba770b731dac93e0819c1bbadb68671b0124260603cf + helpers: ghcr.io/contextforge-org/cf-integration-helpers@sha256:f7e557e263737328225d827e73b519d0b40facb2698c55ee3d2de0ed82f00fa3 + locust: mirror.gcr.io/locustio/locust@sha256:fd39232c31971fe7509582717a95592f0fbf9a917b705f355bd7471e03bdd649 + postgres: postgres@sha256:4ef4dbc939d61acea57712655ddb4b4ab27419c913f94cca0cd57cb3ea3c2280 + redis: mirror.gcr.io/library/redis@sha256:a7859ed111db3c1f5404a973a4747505d559fb5ca32d37e447afc0ef845a2103 +workload: + protocol_version: 2026-07-28 + user_levels: [125, 250, 500, 1000] + first_users: 125 + maximum_users: 1000 + ramp_seconds: 30 + warmup_seconds: 30 + measure_seconds: 3600 + repetitions: 1 + maximum_campaign_seconds: 36000 + plateau_improvement_percent: 5.0 + boundary_percent: 12.5 + config_cache_seconds: 60 + helper_cpu_percent: 70.0 + helper_memory_percent: 80.0 + worker_core_percent: 85.0 + tools: + - convert_time + - echo + - get_stats + - get_system_time + - schema_success + - verify-protocol +scenarios: + - { id: comparison-2v2, label: Built-in dataplane vs external dataplane — 2 vCPU / 2 GB, replicas: 1, cpu: 2, memory_gb: 2, multiplier: 1 } diff --git a/benchmarks/fyre/deploy/builtin.compose.yaml b/benchmarks/fyre/deploy/builtin.compose.yaml new file mode 100644 index 0000000..e806f57 --- /dev/null +++ b/benchmarks/fyre/deploy/builtin.compose.yaml @@ -0,0 +1,142 @@ +services: + network: + image: ${HELPERS_IMAGE:?Set HELPERS_IMAGE to a pinned digest} + restart: unless-stopped + entrypoint: ["/bin/sh", "-c"] + command: ["exec sleep infinity"] + ports: ["${TARGET_BIND_IP:?Set TARGET_BIND_IP to the target private IP}:4444:4444"] + + postgres: + image: ${POSTGRES_IMAGE:?Set POSTGRES_IMAGE to a pinned digest} + restart: unless-stopped + shm_size: 256m + command: + - postgres + - -c + - max_connections=300 + - -c + - shared_buffers=256MB + - -c + - synchronous_commit=off + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD} + POSTGRES_DB: mcp + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres -d mcp"] + interval: 2s + timeout: 3s + retries: 90 + volumes: ["builtin-postgres:/var/lib/postgresql"] + + redis: + image: ${REDIS_IMAGE:?Set REDIS_IMAGE to a pinned digest} + restart: unless-stopped + command: ["redis-server", "--save", "", "--appendonly", "no", "--maxmemory", "512mb", "--maxmemory-policy", "allkeys-lru"] + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 2s + timeout: 2s + retries: 60 + + migration: + image: ${CONTROLPLANE_IMAGE:?Set CONTROLPLANE_IMAGE to a pinned digest} + restart: "no" + entrypoint: ["python3", "-m", "mcpgateway.bootstrap_db"] + environment: &builtin-environment + DATABASE_URL: postgresql+psycopg://postgres:${POSTGRES_PASSWORD}@postgres:5432/mcp + REDIS_URL: redis://redis:6379/0 + CACHE_TYPE: redis + JWT_ALGORITHM: HS256 + JWT_SECRET_KEY: ${JWT_SECRET_KEY:?Set JWT_SECRET_KEY} + JWT_AUDIENCE: mcpgateway-api + JWT_ISSUER: mcpgateway + AUTH_ENCRYPTION_SECRET: ${AUTH_ENCRYPTION_SECRET:?Set AUTH_ENCRYPTION_SECRET} + DEFAULT_USER_PASSWORD: ${DEFAULT_USER_PASSWORD:?Set DEFAULT_USER_PASSWORD} + PLATFORM_ADMIN_EMAIL: admin@example.com + PLATFORM_ADMIN_PASSWORD: ${PLATFORM_ADMIN_PASSWORD:?Set PLATFORM_ADMIN_PASSWORD} + PASSWORD_CHANGE_ENFORCEMENT_ENABLED: "false" + AUTH_REQUIRED: "true" + MCP_CLIENT_AUTH_ENABLED: "true" + MCP_REQUIRE_AUTH: "true" + REQUIRE_USER_IN_DB: "false" + LOG_LEVEL: WARNING + depends_on: + postgres: + condition: service_healthy + + gateway: + image: ${CONTROLPLANE_IMAGE:?Set CONTROLPLANE_IMAGE to a pinned digest} + restart: unless-stopped + network_mode: service:network + read_only: true + tmpfs: + - /tmp:size=128M,mode=1777 + - /var/tmp:size=32M,mode=1777 + - /run:size=8M,mode=0755 + ulimits: + nofile: + soft: 65536 + hard: 65536 + environment: + <<: *builtin-environment + HOST: 0.0.0.0 + PORT: "4444" + TRANSPORT_TYPE: streamablehttp + RUST_MCP_MODE: "off" + MCPGATEWAY_SKIP_MIGRATIONS: "true" + GATEWAY_TOOL_NAME_SEPARATOR: _ + SSRF_ALLOW_LOCALHOST: "true" + SSRF_ALLOW_PRIVATE_NETWORKS: "true" + SSRF_DNS_FAIL_CLOSED: "false" + PLUGINS_ENABLED: "false" + MCPGATEWAY_CATALOG_ENABLED: "false" + MCPGATEWAY_UI_ENABLED: "false" + MCPGATEWAY_ADMIN_API_ENABLED: "true" + ENABLE_METRICS: "false" + DB_METRICS_RECORDING_ENABLED: "false" + STRUCTURED_LOGGING_DATABASE_ENABLED: "false" + AUDIT_TRAIL_ENABLED: "false" + SECURITY_LOGGING_ENABLED: "false" + DISABLE_ACCESS_LOG: "true" + COMPRESSION_ENABLED: "false" + VALIDATION_MIDDLEWARE_ENABLED: "false" + CORRELATION_ID_ENABLED: "false" + OBSERVABILITY_ENABLED: "false" + RATE_LIMITING_ENABLED: "false" + GUNICORN_WORKERS: "4" + GUNICORN_KEEP_ALIVE: "30" + GUNICORN_BACKLOG: "4096" + DB_POOL_CLASS: queue + DB_POOL_SIZE: "15" + DB_MAX_OVERFLOW: "10" + DB_POOL_PRE_PING: "true" + HTTPX_MAX_CONNECTIONS: "1000" + HTTPX_MAX_KEEPALIVE_CONNECTIONS: "500" + MCP_SESSION_POOL_ENABLED: "true" + MCP_SESSION_POOL_MAX_PER_KEY: "1000" + TOOL_RATE_LIMIT: "600000" + TOOL_CONCURRENT_LIMIT: "5000" + depends_on: + migration: + condition: service_completed_successfully + redis: + condition: service_healthy + healthcheck: + test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:4444/health', timeout=3)"] + interval: 3s + timeout: 5s + retries: 100 + + admin: + profiles: ["helpers"] + image: ${CONTROLPLANE_IMAGE:?Set CONTROLPLANE_IMAGE to a pinned digest} + network_mode: service:network + entrypoint: ["python3", "/work/register_builtin.py"] + volumes: ["./register_builtin.py:/work/register_builtin.py:ro"] + environment: + <<: *builtin-environment + GATEWAY_URL: http://127.0.0.1:4444 + +volumes: + builtin-postgres: diff --git a/benchmarks/fyre/deploy/dataplane.compose.yaml b/benchmarks/fyre/deploy/dataplane.compose.yaml new file mode 100644 index 0000000..2a26d73 --- /dev/null +++ b/benchmarks/fyre/deploy/dataplane.compose.yaml @@ -0,0 +1,72 @@ +services: + network: + image: ${HELPERS_IMAGE:?Set HELPERS_IMAGE to a pinned digest} + restart: unless-stopped + entrypoint: ["/bin/sh", "-c"] + command: ["exec sleep infinity"] + ports: ["${TARGET_BIND_IP:?Set TARGET_BIND_IP to the target private IP}:4445:4445"] + + redis: + image: ${REDIS_IMAGE:?Set REDIS_IMAGE to a pinned digest} + restart: unless-stopped + command: ["redis-server", "--save", "", "--appendonly", "no"] + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 2s + timeout: 2s + retries: 60 + + dataplane: + image: ${DATAPLANE_IMAGE:?Set DATAPLANE_IMAGE to a pinned digest} + restart: unless-stopped + network_mode: service:network + ulimits: + nofile: + soft: 65536 + hard: 65536 + environment: + CONTEXTFORGE_DATA_PLANE_ADDRESS: 0.0.0.0:4445 + CONTEXTFORGE_DATA_PLANE_REDIS_HOSTNAME: redis + CONTEXTFORGE_DATA_PLANE_REDIS_PORT: "6379" + CONTEXTFORGE_DATA_PLANE_REDIS_CONNECTION_MODE: plain-text + CONTEXTFORGE_DATA_PLANE_JWKS_URL: http://127.0.0.1:4446/.well-known/jwks.json + CONTEXTFORGE_DATA_PLANE_UPSTREAM_CONNECTION_MODE: plain-text-or-tls + CONTEXTFORGE_GATEWAY_RS_MCP_ALLOWED_HOSTS: ${DATAPLANE_ALLOWED_HOSTS:?Set DATAPLANE_ALLOWED_HOSTS} + CONTEXTFORGE_DATA_PLANE_USER_CONFIG_CACHE_EXPIRY_SECONDS: ${CONFIG_CACHE_SECONDS:-60} + RUST_LOG: warn + depends_on: + network: + condition: service_started + redis: + condition: service_healthy + + auth: + image: ${HELPERS_IMAGE:?Set HELPERS_IMAGE to a pinned digest} + restart: unless-stopped + network_mode: service:network + volumes: ["./state/keys:/keys"] + command: ["__helper", "auth"] + healthcheck: + test: ["CMD", "cf-integration", "__helper", "health"] + interval: 2s + timeout: 2s + retries: 60 + depends_on: + network: + condition: service_started + dataplane: + condition: service_started + + config_writer: + profiles: ["helpers"] + image: ${HELPERS_IMAGE:?Set HELPERS_IMAGE to a pinned digest} + network_mode: service:network + volumes: ["./state/keys:/keys"] + environment: + CF_CONFIG_REDIS_URL: redis://redis:6379 + entrypoint: ["cf-integration", "__helper"] + depends_on: + network: + condition: service_started + redis: + condition: service_healthy diff --git a/benchmarks/fyre/deploy/fast-time.compose.yaml b/benchmarks/fyre/deploy/fast-time.compose.yaml new file mode 100644 index 0000000..8ac318f --- /dev/null +++ b/benchmarks/fyre/deploy/fast-time.compose.yaml @@ -0,0 +1,19 @@ +services: + fast_time: + image: ${FAST_TIME_IMAGE:?Set FAST_TIME_IMAGE to a pinned digest} + restart: unless-stopped + network_mode: host + command: [] + environment: + BIND_ADDRESS: ${FAST_TIME_BIND_IP:?Set FAST_TIME_BIND_IP to the backend private IP}:9080 + RUST_LOG: warn + ulimits: + nofile: + soft: 65536 + hard: 65536 + healthcheck: + test: ["CMD-SHELL", "curl -fsS http://${FAST_TIME_BIND_IP}:9080/health"] + interval: 2s + timeout: 2s + retries: 60 + start_period: 2s diff --git a/benchmarks/fyre/deploy/monitor.py b/benchmarks/fyre/deploy/monitor.py new file mode 100644 index 0000000..67ec236 --- /dev/null +++ b/benchmarks/fyre/deploy/monitor.py @@ -0,0 +1,168 @@ +"""Sample Linux host pressure as JSON lines without third-party packages.""" + +from __future__ import annotations + +import argparse +import json +import os +import platform +import socket +import subprocess +import time +from pathlib import Path + + +def read(path: str) -> str: + try: + return Path(path).read_text(encoding="utf-8", errors="replace") + except OSError as error: + return f"error={error}" + + +def cpu_times() -> dict[str, list[int]]: + rows: dict[str, list[int]] = {} + for line in read("/proc/stat").splitlines(): + fields = line.split() + if fields and fields[0].startswith("cpu"): + rows[fields[0]] = [int(value) for value in fields[1:]] + return rows + + +def cpu_percent( + previous: dict[str, list[int]], current: dict[str, list[int]] +) -> dict[str, dict[str, float]]: + result = {} + for name, values in current.items(): + before = previous.get(name, values) + deltas = [max(0, after - old) for old, after in zip(before, values)] + total = sum(deltas) or 1 + idle = sum(deltas[index] for index in (3, 4) if index < len(deltas)) + steal = deltas[7] if len(deltas) > 7 else 0 + result[name] = { + "busy_percent": round(100.0 * (total - idle) / total, 3), + "steal_percent": round(100.0 * steal / total, 3), + } + return result + + +def memory() -> dict[str, int | float]: + values = {} + for line in read("/proc/meminfo").splitlines(): + key, _, rest = line.partition(":") + try: + values[key] = int(rest.split()[0]) + except (IndexError, ValueError): + continue + total = values.get("MemTotal", 0) + available = values.get("MemAvailable", 0) + return { + "total_kib": total, + "available_kib": available, + "used_percent": round(100.0 * (total - available) / total, 3) if total else 0.0, + "swap_total_kib": values.get("SwapTotal", 0), + "swap_free_kib": values.get("SwapFree", 0), + } + + +def command_output(command: list[str]) -> str: + try: + return subprocess.run( + command, check=False, text=True, capture_output=True, timeout=3 + ).stdout.strip() + except (OSError, subprocess.TimeoutExpired) as error: + return f"error={error}" + + +def docker_state() -> str: + container_ids = [ + container_id + for container_id in command_output( + ["docker", "ps", "--all", "--quiet"] + ).splitlines() + if container_id and not container_id.startswith("error=") + ] + if not container_ids: + return "[]" + return command_output( + [ + "docker", + "inspect", + "--format", + "{{json .State}}", + *container_ids, + ] + ) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--output", required=True) + parser.add_argument("--role", required=True) + parser.add_argument("--interval", type=float, default=1.0) + args = parser.parse_args() + output = Path(args.output) + output.parent.mkdir(parents=True, exist_ok=True) + previous = cpu_times() + with output.open("a", encoding="utf-8", buffering=1) as stream: + stream.write( + json.dumps( + { + "kind": "host", + "role": args.role, + "hostname": socket.gethostname(), + "cpu_model": next( + ( + line.partition(":")[2].strip() + for line in read("/proc/cpuinfo").splitlines() + if line.startswith("model name") + ), + platform.processor(), + ), + "logical_cpus": os.cpu_count(), + "kernel": platform.release(), + }, + sort_keys=True, + ) + + "\n" + ) + while True: + time.sleep(args.interval) + current = cpu_times() + stream.write( + json.dumps( + { + "kind": "sample", + "time": time.time(), + "cpu": cpu_percent(previous, current), + "memory": memory(), + "loadavg": read("/proc/loadavg").strip(), + "pressure_cpu": read("/proc/pressure/cpu").strip(), + "pressure_memory": read("/proc/pressure/memory").strip(), + "vmstat": read("/proc/vmstat").strip(), + "sockstat": read("/proc/net/sockstat").strip(), + "netstat": read("/proc/net/netstat").strip(), + "snmp": read("/proc/net/snmp").strip(), + "network": read("/proc/net/dev").strip(), + "ss": command_output(["ss", "-s"]), + "processes": command_output( + [ + "ps", + "-eo", + "pid,ppid,comm,%cpu,%mem,rss,vsz,stat", + "--sort=-%cpu", + ] + ), + "docker": command_output( + ["docker", "stats", "--no-stream", "--format", "{{json .}}"] + ), + "docker_state": docker_state(), + }, + sort_keys=True, + ) + + "\n" + ) + previous = current + + +if __name__ == "__main__": + main() diff --git a/benchmarks/fyre/deploy/register_builtin.py b/benchmarks/fyre/deploy/register_builtin.py new file mode 100644 index 0000000..abe406a --- /dev/null +++ b/benchmarks/fyre/deploy/register_builtin.py @@ -0,0 +1,159 @@ +"""Register the remote Fast Time server and print benchmark credentials as JSON.""" + +from __future__ import annotations + +import argparse +import json +import os +import time +import urllib.error +import urllib.request +import uuid + +import jwt + +SERVER_ID = "9779b6698cbd4b4995ee04a4fab38737" +EXPECTED_TOOLS = { + "convert_time", + "echo", + "get_stats", + "get_system_time", + "schema_success", + "verify-protocol", +} + + +def token() -> str: + now = int(time.time()) + email = "admin@example.com" + payload = { + "username": email, + "sub": email, + "iat": now, + "exp": now + 43_200, + "iss": "mcpgateway", + "aud": "mcpgateway-api", + "jti": str(uuid.uuid4()), + "env": "development", + "user": { + "email": email, + "full_name": "FYRE Benchmark", + "is_admin": True, + "auth_provider": "cli", + }, + "teams": None, + } + return jwt.encode(payload, os.environ["JWT_SECRET_KEY"], algorithm="HS256") + + +def api(method: str, path: str, bearer: str, data: dict | None = None): + request = urllib.request.Request( + os.environ.get("GATEWAY_URL", "http://127.0.0.1:4444").rstrip("/") + path, + method=method, + data=json.dumps(data).encode() if data is not None else None, + headers={ + "Authorization": f"Bearer {bearer}", + "Content-Type": "application/json", + }, + ) + with urllib.request.urlopen(request, timeout=60) as response: + body = response.read() + return json.loads(body) if body else None + + +def retry(method: str, path: str, bearer: str, data: dict | None = None): + last: Exception | None = None + for _attempt in range(60): + try: + return api(method, path, bearer, data) + except (OSError, urllib.error.HTTPError) as error: + last = error + time.sleep(2) + raise RuntimeError(f"{method} {path} did not become ready") from last + + +def tool_base(name: str) -> str: + for prefix in ("fast_time_", "fast-time-"): + if name.startswith(prefix): + name = name[len(prefix) :] + break + return "verify-protocol" if name == "verify_protocol" else name + + +def tool_identity(tool: dict) -> str: + for field in ("originalName", "customName", "name"): + name = tool.get(field) + if isinstance(name, str) and tool_base(name) in EXPECTED_TOOLS: + return tool_base(name) + return "" + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--backend", required=True) + args = parser.parse_args() + bearer = token() + gateways = retry("GET", "/gateways", bearer) or [] + gateway = next((item for item in gateways if item.get("name") == "fast_time"), None) + if gateway is None: + gateway = retry( + "POST", + "/gateways", + bearer, + {"name": "fast_time", "url": args.backend, "transport": "STREAMABLEHTTP"}, + ) + gateway_id = gateway["id"] + retry( + "POST", + f"/gateways/{gateway_id}/tools/refresh?include_resources=true&include_prompts=true", + bearer, + ) + selected = [] + all_tools = [] + for _attempt in range(60): + all_tools = retry("GET", "/tools", bearer) or [] + selected = [ + item + for item in all_tools + if (item.get("gatewayId") or item.get("gateway_id")) == gateway_id + and tool_identity(item) in EXPECTED_TOOLS + ] + if {tool_identity(item) for item in selected} == EXPECTED_TOOLS: + break + time.sleep(1) + else: + raise RuntimeError("Fast Time registration did not expose all six benchmark tools") + try: + api("DELETE", f"/servers/{SERVER_ID}", bearer) + except urllib.error.HTTPError as error: + if error.code != 404: + raise + retry( + "POST", + "/servers", + bearer, + { + "server": { + "id": SERVER_ID, + "name": "Fast Time Server", + "description": "FYRE zero-delay Fast Time benchmark", + "associated_tools": [item["id"] for item in selected], + "associated_resources": [], + "associated_prompts": [], + } + }, + ) + print( + json.dumps( + { + "token": bearer, + "server_id": SERVER_ID, + "tool_names": sorted(item["name"] for item in selected), + }, + separators=(",", ":"), + ) + ) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/fyre/deploy/run_locust.py b/benchmarks/fyre/deploy/run_locust.py new file mode 100644 index 0000000..30bf0a1 --- /dev/null +++ b/benchmarks/fyre/deploy/run_locust.py @@ -0,0 +1,225 @@ +"""Run one distributed, headless Locust phase and propagate any worker failure.""" + +from __future__ import annotations + +import argparse +import os +import signal +import subprocess +import sys +import time +from pathlib import Path + +CONTAINERS: list[str] = [] +NETWORKS: list[str] = [] + + +def docker( + *arguments: str, check: bool = True, capture: bool = False +) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ["docker", *arguments], + check=check, + text=True, + stdout=subprocess.PIPE if capture else None, + stderr=subprocess.STDOUT if capture else None, + ) + + +def cleanup() -> None: + if CONTAINERS: + docker("rm", "--force", *CONTAINERS, check=False, capture=True) + if NETWORKS: + docker("network", "rm", *NETWORKS, check=False, capture=True) + + +def collect_container_logs(output: Path) -> None: + for name in CONTAINERS: + result = docker("logs", name, check=False, capture=True) + if isinstance(result.stdout, str) and result.stdout: + output.joinpath(f"{name}.docker.log").write_text( + result.stdout, encoding="utf-8" + ) + + +def stop(_signal: int, _frame) -> None: + cleanup() + raise SystemExit(130) + + +def container_state(name: str) -> tuple[str, int]: + result = docker( + "inspect", + "--format", + "{{.State.Status}} {{.State.ExitCode}}", + name, + check=False, + capture=True, + ) + if result.returncode != 0: + return "missing", 1 + fields = result.stdout.strip().split() + if len(fields) != 2: + return "invalid", 1 + try: + return fields[0], int(fields[1]) + except ValueError: + return "invalid", 1 + + +def wait_for_cluster(master: str, workers: list[str]) -> int: + startup_attempts = 60 + while True: + master_state, master_exit = container_state(master) + if master_state in {"exited", "dead", "missing", "invalid"}: + return master_exit + if master_state == "created" and startup_attempts: + startup_attempts -= 1 + time.sleep(0.5) + continue + pending = False + for worker in workers: + worker_state, worker_exit = container_state(worker) + if worker_state in {"exited", "dead"} and worker_exit == 0: + continue + if worker_state == "created" and startup_attempts: + pending = True + continue + if worker_state != "running": + docker("stop", "--time", "1", master, check=False, capture=True) + return 1 + if pending: + startup_attempts -= 1 + time.sleep(0.5) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--image", required=True) + parser.add_argument("--users", type=int, required=True) + parser.add_argument("--spawn-rate", type=float, required=True) + parser.add_argument("--seconds", type=int, required=True) + parser.add_argument("--workers", type=int, required=True) + parser.add_argument("--output", required=True) + parser.add_argument("--env-file", required=True) + parser.add_argument("--reset-stats", action="store_true") + parser.add_argument("--measurement-seconds", type=int) + parser.add_argument("--warmup-seconds", type=int) + args = parser.parse_args() + if min(args.users, args.spawn_rate, args.seconds, args.workers) <= 0: + parser.error("users, spawn-rate, seconds, and workers must be positive") + if args.measurement_seconds is not None and args.measurement_seconds <= 0: + parser.error("measurement-seconds must be positive") + if args.warmup_seconds is not None and args.warmup_seconds < 0: + parser.error("warmup-seconds cannot be negative") + measurement = args.measurement_seconds is not None + if args.reset_stats != measurement or (args.warmup_seconds is not None) != measurement: + parser.error( + "reset-stats, measurement-seconds, and warmup-seconds must be used together" + ) + + signal.signal(signal.SIGINT, stop) + signal.signal(signal.SIGTERM, stop) + output = Path(args.output) + output.mkdir(parents=True, exist_ok=True) + prefix = f"cf-fyre-{os.getpid()}" + master = f"{prefix}-master" + network = f"{prefix}-network" + docker("network", "create", network, capture=True) + NETWORKS.append(network) + CONTAINERS.append(master) + common = [ + "--user", + "0:0", + "--network", + network, + "--ulimit", + "nofile=65536:65536", + "--env-file", + args.env_file, + "--volume", + f"{Path.cwd() / 'locustfile_mcp.py'}:/mnt/locust-cf/locustfile_mcp.py:ro", + "--volume", + f"{output.resolve()}:/mnt/reports", + ] + if args.reset_stats: + common.extend( + [ + "--env", + "MCP_MEASUREMENT_MARKER=/mnt/reports/measurement-start.txt", + "--env", + f"MCP_MEASUREMENT_SECONDS={args.measurement_seconds}", + "--env", + f"MCP_WARMUP_SECONDS={args.warmup_seconds}", + ] + ) + run_seconds = args.seconds + 30 if args.measurement_seconds is not None else args.seconds + master_args = [ + "run", + "--detach", + "--name", + master, + *common, + args.image, + "-f", + "/mnt/locust-cf/locustfile_mcp.py", + "--master", + "--expect-workers", + str(args.workers), + "--headless", + "--users", + str(args.users), + "--spawn-rate", + str(args.spawn_rate), + "--run-time", + f"{run_seconds}s", + "--stop-timeout", + "1", + "--host", + "http://127.0.0.1", + "--csv", + "/mnt/reports/locust", + "--csv-full-history", + "--html", + "/mnt/reports/locust.html", + "--json-file", + "/mnt/reports/locust.json", + "--logfile", + "/mnt/reports/locust.log", + ] + docker(*master_args) + try: + workers = [] + for index in range(args.workers): + name = f"{prefix}-worker-{index + 1}" + CONTAINERS.append(name) + workers.append(name) + docker( + "run", + "--detach", + "--name", + name, + *common, + "--env", + f"MCP_REPLICA_OFFSET={index}", + args.image, + "-f", + "/mnt/locust-cf/locustfile_mcp.py", + "--worker", + "--master-host", + master, + ) + status = wait_for_cluster(master, workers) + for name in CONTAINERS: + state, exit_code = container_state(name) + if state in {"exited", "dead"} and exit_code != 0: + status = 1 + sys.exit(status) + finally: + time.sleep(0.2) + collect_container_logs(output) + cleanup() + + +if __name__ == "__main__": + main() diff --git a/benchmarks/fyre/deploy/smoke.py b/benchmarks/fyre/deploy/smoke.py new file mode 100644 index 0000000..6c35d24 --- /dev/null +++ b/benchmarks/fyre/deploy/smoke.py @@ -0,0 +1,100 @@ +"""Call every measured Fast Time tool through every dataplane replica.""" + +from __future__ import annotations + +import argparse +import json +import urllib.error +import urllib.request +import uuid + +TOOLS = { + "convert_time": { + "time": "2025-06-21T16:00:00Z", + "source_timezone": "UTC", + "target_timezone": "Europe/Dublin", + }, + "echo": {"message": "cf-integration", "delay": 0}, + "get_stats": {}, + "get_system_time": {"timezone": "UTC"}, + "schema_success": {}, + "verify-protocol": {}, +} + + +def base_tool_name(name: str) -> str: + for prefix in ("fast_time_", "fast-time-"): + if name.startswith(prefix): + name = name[len(prefix) :] + break + return "verify-protocol" if name == "verify_protocol" else name + + +def call(url: str, token: str, tool: str, arguments: dict) -> None: + payload = { + "jsonrpc": "2.0", + "id": str(uuid.uuid4()), + "method": "tools/call", + "params": { + "name": tool, + "arguments": arguments, + "_meta": { + "io.modelcontextprotocol/protocolVersion": "2026-07-28", + "io.modelcontextprotocol/clientInfo": { + "name": "cf-integration-smoke", + "version": "1.0", + }, + "io.modelcontextprotocol/clientCapabilities": {}, + }, + }, + } + request = urllib.request.Request( + url, + data=json.dumps(payload).encode(), + headers={ + "Accept": "application/json, text/event-stream", + "Authorization": f"Bearer {token}", + "Content-Type": "application/json", + "Mcp-Protocol-Version": "2026-07-28", + "Mcp-Method": "tools/call", + "Mcp-Name": tool, + }, + ) + try: + with urllib.request.urlopen(request, timeout=10) as response: + body = response.read().decode() + if ( + response.status != 200 + or '"error"' in body + or '"isError":true' in body.replace(" ", "") + ): + raise RuntimeError( + f"{url} {tool} failed: HTTP {response.status}: {body[:500]}" + ) + except urllib.error.HTTPError as error: + body = error.read().decode(errors="replace") + raise RuntimeError( + f"{url} {tool} failed: HTTP {error.code}: {body[:500]}" + ) from error + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--urls", required=True) + parser.add_argument("--token-file", required=True) + parser.add_argument("--tool-names", default=",".join(TOOLS)) + args = parser.parse_args() + with open(args.token_file, encoding="utf-8") as stream: + token = stream.read().strip() + tool_names = [name.strip() for name in args.tool_names.split(",") if name.strip()] + if {base_tool_name(name) for name in tool_names} != set(TOOLS): + raise RuntimeError("smoke requires exactly the six Fast Time benchmark tools") + for url in args.urls.split(","): + for tool in tool_names: + arguments = dict(TOOLS[base_tool_name(tool)]) + call(url, token, tool, arguments) + print(f"PASS {url} {tool}") + + +if __name__ == "__main__": + main() diff --git a/benchmarks/fyre/report.py b/benchmarks/fyre/report.py new file mode 100644 index 0000000..b8f37c7 --- /dev/null +++ b/benchmarks/fyre/report.py @@ -0,0 +1,404 @@ +"""Build machine-readable and Slack-ready FYRE benchmark reports.""" + +from __future__ import annotations + +import argparse +import csv +import json +from pathlib import Path + + +def comparison_report(config: dict, results_root: Path, *, render: bool = True) -> None: + result_path = results_root / "comparison" / "result.json" + if not result_path.is_file(): + raise RuntimeError("comparison result is required") + result = json.loads(result_path.read_text(encoding="utf-8")) + if result.get("status") != "confirmed": + raise RuntimeError("all eight comparison benchmarks must pass before reporting") + lanes = { + lane: {item["users"]: item for item in result["runs"][lane]} + for lane in ("builtin", "rust") + } + rows = [] + for users in config["workload"]["user_levels"]: + builtin = lanes["builtin"][users] + rust = lanes["rust"][users] + rows.append( + { + "users": users, + "built_in_dataplane_requests": builtin["requests"], + "built_in_dataplane_errors": builtin["failures"], + "built_in_dataplane_rps": builtin["rps"], + "built_in_dataplane_p50_ms": builtin["p50_ms"], + "built_in_dataplane_p95_ms": builtin["p95_ms"], + "built_in_dataplane_p99_ms": builtin["p99_ms"], + "external_dataplane_requests": rust["requests"], + "external_dataplane_errors": rust["failures"], + "external_dataplane_rps": rust["rps"], + "external_dataplane_p50_ms": rust["p50_ms"], + "external_dataplane_p95_ms": rust["p95_ms"], + "external_dataplane_p99_ms": rust["p99_ms"], + "external_vs_built_in": rust["rps"] / builtin["rps"], + } + ) + (results_root / "summary.json").write_text( + json.dumps({"rows": rows}, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + with (results_root / "summary.csv").open("w", newline="", encoding="utf-8") as stream: + writer = csv.DictWriter(stream, fieldnames=rows[0].keys()) + writer.writeheader() + writer.writerows(rows) + + if not render: + return + + import matplotlib.pyplot as plt + from matplotlib.patches import FancyBboxPatch + + helpers = config["active_helper"] + workload = config["workload"] + target = config["scenarios"][0] + figure = plt.figure(figsize=(18, 10), dpi=160, facecolor="#0b1020") + axis = figure.add_axes([0, 0, 1, 1]) + axis.set_axis_off() + figure.text( + 0.035, + 0.95, + "FYRE built-in dataplane vs external dataplane — one-hour load comparison", + color="white", + fontsize=25, + fontweight="bold", + ) + figure.text( + 0.035, + 0.91, + "Eight zero-error benchmarks • same modern client and same target VM allocation • private FYRE network", + color="#a7b0c0", + fontsize=12, + ) + + cards = [ + ( + 0.035, + "LOAD GENERATOR", + f"Locust VM • {helpers['locust_cpu']} vCPU / {helpers['locust_memory_gb']} GB\n" + f"{max(2, int(helpers['locust_cpu']) - 1)} distributed workers • zero wait", + ), + ( + 0.355, + "TARGET — SAME VM, SEQUENTIAL", + f"{target['cpu']} vCPU / {target['memory_gb']} GB\n" + "Built-in dataplane: Python gateway + Postgres + Redis\n" + "External dataplane: Rust + Redis + loopback JWKS", + ), + ( + 0.71, + "BACKEND", + f"Fast Time VM • {helpers['fast_time_cpu']} vCPU / {helpers['fast_time_memory_gb']} GB\n6 tools • explicit zero delay", + ), + ] + widths = [0.27, 0.31, 0.255] + for (x, title, body), width in zip(cards, widths): + axis.add_patch( + FancyBboxPatch( + (x, 0.74), + width, + 0.12, + boxstyle="round,pad=0.008,rounding_size=0.008", + linewidth=1.4, + edgecolor="#3a4765", + facecolor="#172039", + ) + ) + figure.text(x + 0.018, 0.825, title, color="#49a7ff", fontsize=11, fontweight="bold") + figure.text(x + 0.018, 0.78, body, color="white", fontsize=10, va="center", linespacing=1.4) + for start, end in ((0.305, 0.355), (0.665, 0.71)): + axis.annotate( + "", + xy=(end, 0.8), + xytext=(start, 0.8), + arrowprops={"arrowstyle": "-|>", "color": "#45d6a0", "lw": 2.5}, + ) + + table_axis = figure.add_axes([0.025, 0.28, 0.95, 0.38]) + table_axis.axis("off") + headers = [ + "Users", + "Built-in DP\nrequests", + "Built-in DP\nerrors", + "Built-in DP\nRPS", + "Built-in DP p50 /\np95 / p99", + "External DP\nrequests", + "External DP\nerrors", + "External DP\nRPS", + "External DP p50 /\np95 / p99", + "External vs\nbuilt-in", + ] + cells = [ + [ + f"{row['users']:,}", + f"{row['built_in_dataplane_requests']:,}", + str(row["built_in_dataplane_errors"]), + f"{row['built_in_dataplane_rps']:,.2f}", + f"{row['built_in_dataplane_p50_ms']:.0f} / {row['built_in_dataplane_p95_ms']:.0f} / {row['built_in_dataplane_p99_ms']:.0f} ms", + f"{row['external_dataplane_requests']:,}", + str(row["external_dataplane_errors"]), + f"{row['external_dataplane_rps']:,.2f}", + f"{row['external_dataplane_p50_ms']:.0f} / {row['external_dataplane_p95_ms']:.0f} / {row['external_dataplane_p99_ms']:.0f} ms", + f"{row['external_vs_built_in']:.2f}×", + ] + for row in rows + ] + table = table_axis.table( + cellText=cells, + colLabels=headers, + cellLoc="center", + loc="center", + colWidths=[0.06, 0.105, 0.065, 0.09, 0.15, 0.105, 0.065, 0.09, 0.15, 0.10], + ) + table.auto_set_font_size(False) + table.set_fontsize(9.5) + table.scale(1, 2.3) + for (row, _column), cell in table.get_celld().items(): + cell.set_edgecolor("#34415f") + cell.set_facecolor("#172039" if row else "#253250") + cell.get_text().set_color("white") + if row == 0: + cell.get_text().set_fontweight("bold") + + figure.text( + 0.035, + 0.20, + f"Method: MCP {workload['protocol_version']} for both clients • FastHttpUser • " + f"{workload['ramp_seconds']} s ramp • {workload['warmup_seconds']} s warmup • " + f"{workload['measure_seconds'] // 60} min measured • statistics reset after warmup", + color="#a7b0c0", + fontsize=11, + ) + figure.text( + 0.035, + 0.155, + "Traffic: the same six Fast Time tools through each public MCP route • first request or worker error stops the campaign", + color="#a7b0c0", + fontsize=11, + ) + figure.text( + 0.035, + 0.095, + "External vs built-in = external dataplane RPS ÷ built-in dataplane RPS at the same user count.", + color="#45d6a0", + fontsize=13, + fontweight="bold", + ) + for name in ("slack-comparison.png", "slack-scaling.png"): + figure.savefig( + results_root / name, + bbox_inches="tight", + facecolor=figure.get_facecolor(), + ) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--config", required=True) + parser.add_argument("--results", required=True) + args = parser.parse_args() + config = json.loads(Path(args.config).read_text(encoding="utf-8")) + results_root = Path(args.results) + if config.get("benchmark_kind", "scaling") == "comparison": + comparison_report(config, results_root) + return + import matplotlib.pyplot as plt + from matplotlib.patches import FancyBboxPatch + + results = {} + for scenario in config["scenarios"]: + path = results_root / scenario["id"] / "result.json" + if path.exists(): + results[scenario["id"]] = json.loads(path.read_text(encoding="utf-8")) + if "baseline" not in results: + raise RuntimeError("baseline result is required to calculate scaling") + + baseline = results["baseline"]["rps"] + rows = [] + for scenario in config["scenarios"]: + result = results.get(scenario["id"]) + if not result or result.get("status") != "confirmed": + continue + total_cpu = scenario["replicas"] * scenario["cpu"] + speedup = result["rps"] / baseline + row = { + "scenario": scenario["label"], + "scenario_id": scenario["id"], + "replicas": scenario["replicas"], + "cpu_per_vm": scenario["cpu"], + "memory_gb_per_vm": scenario["memory_gb"], + "total_cpu": total_cpu, + "total_memory_gb": scenario["replicas"] * scenario["memory_gb"], + "users": result["users"], + "lower_bound": result.get("lower_bound", False), + "rps": result["rps"], + "p50_ms": result["p50_ms"], + "p95_ms": result["p95_ms"], + "p99_ms": result["p99_ms"], + "speedup": speedup, + "efficiency": speedup / scenario["multiplier"], + "rps_per_vcpu": result["rps"] / total_cpu, + "rps_cv_percent": result["rps_cv_percent"], + "horizontal_advantage": None, + "replica_imbalance_percent": result["replica_imbalance_percent"], + } + if scenario["id"].startswith("horizontal-"): + vertical_id = scenario["id"].replace("horizontal", "vertical") + vertical = results.get(vertical_id) + if vertical and vertical.get("status") == "confirmed": + row["horizontal_advantage"] = result["rps"] / vertical["rps"] + rows.append(row) + + (results_root / "summary.json").write_text( + json.dumps({"rows": rows}, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + with (results_root / "summary.csv").open( + "w", newline="", encoding="utf-8" + ) as stream: + writer = csv.DictWriter(stream, fieldnames=rows[0].keys()) + writer.writeheader() + writer.writerows(rows) + + helpers = config["active_helper"] + workload = config["workload"] + architecture = ( + f"Architecture: Locust {helpers['locust_cpu']}c/{helpers['locust_memory_gb']}G VM" + " → Rust dataplane VM(s) → " + f"Fast Time {helpers['fast_time_cpu']}c/{helpers['fast_time_memory_gb']}G VM" + " • private FYRE traffic" + ) + method = ( + f"MCP {workload['protocol_version']} • {len(workload['tools'])} zero-delay tools" + f" • {workload['ramp_seconds']}s ramp • {workload['warmup_seconds']}s warmup" + f" • {workload['measure_seconds']}s measured • " + f"{workload['repetitions']} confirmations • fail-fast on first error" + ) + deployment = ( + "Each dataplane VM: one Rust instance + local Redis + loopback JWKS • " + f"{config['infrastructure']['os']} • direct balanced replica traffic" + ) + + figure = plt.figure(figsize=(16, 10), dpi=160, facecolor="#0b1020") + grid = figure.add_gridspec(2, 1, height_ratios=[2.1, 1.5], hspace=0.2) + axis = figure.add_subplot(grid[0]) + axis.set_facecolor("#0b1020") + colors = [ + "#a7b0c0" + if row["scenario_id"] == "baseline" + else "#49a7ff" + if "vertical" in row["scenario_id"] + else "#45d6a0" + for row in rows + ] + bars = axis.bar( + [row["scenario"] for row in rows], [row["rps"] for row in rows], color=colors + ) + axis.set_ylim(0, max(row["rps"] for row in rows) * 1.18) + axis.set_ylabel("Confirmed zero-error requests/second", color="white", fontsize=12) + axis.tick_params(axis="x", colors="white", rotation=12) + axis.tick_params(axis="y", colors="white") + for spine in axis.spines.values(): + spine.set_color("#56617a") + axis.grid(axis="y", color="#28324a", alpha=0.7) + for bar, row in zip(bars, rows): + comparison = f"{row['speedup']:.2f}x" + if row["horizontal_advantage"] is not None: + comparison += f"\n{row['horizontal_advantage']:.2f}x vs vertical" + axis.text( + bar.get_x() + bar.get_width() / 2, + bar.get_height(), + f"{row['rps']:,.0f} RPS\n{comparison}", + ha="center", + va="bottom", + color="white", + fontsize=10, + fontweight="bold", + ) + figure.suptitle( + "ContextForge Rust dataplane scaling on FYRE", + color="white", + fontsize=20, + fontweight="bold", + x=0.065, + y=0.985, + ha="left", + ) + figure.text( + 0.065, + 0.935, + architecture, + color="#a7b0c0", + fontsize=10, + ha="left", + ) + figure.text(0.065, 0.91, method, color="#a7b0c0", fontsize=10, ha="left") + figure.text( + 0.065, + 0.885, + deployment, + color="#a7b0c0", + fontsize=10, + ha="left", + ) + figure.subplots_adjust(top=0.83) + + table_axis = figure.add_subplot(grid[1]) + table_axis.axis("off") + headers = [ + "Scenario", + "VMs × size", + "Users", + "RPS", + "p50 / p95 / p99 ms", + "Speedup", + "Efficiency", + "RPS/vCPU", + "CV / imbalance", + ] + cells = [] + for row in rows: + users = f"≥{row['users']:,}" if row["lower_bound"] else f"{row['users']:,}" + cells.append( + [ + row["scenario"], + f"{row['replicas']} × {row['cpu_per_vm']}c/{row['memory_gb_per_vm']}G", + users, + f"{row['rps']:,.0f}", + f"{row['p50_ms']:.1f} / {row['p95_ms']:.1f} / {row['p99_ms']:.1f}", + f"{row['speedup']:.2f}x", + f"{100 * row['efficiency']:.1f}%", + f"{row['rps_per_vcpu']:,.0f}", + f"{row['rps_cv_percent']:.1f}% / {row['replica_imbalance_percent']:.1f}%", + ] + ) + table = table_axis.table( + cellText=cells, + colLabels=headers, + cellLoc="center", + loc="center", + colWidths=[0.16, 0.12, 0.08, 0.09, 0.18, 0.09, 0.09, 0.10, 0.07], + ) + table.auto_set_font_size(False) + table.set_fontsize(9) + table.scale(1, 1.8) + for (row, _column), cell in table.get_celld().items(): + cell.set_edgecolor("#28324a") + cell.set_facecolor("#172039" if row else "#253250") + cell.get_text().set_color("white") + if row == 0: + cell.get_text().set_fontweight("bold") + figure.savefig( + results_root / "slack-scaling.png", + bbox_inches="tight", + facecolor=figure.get_facecolor(), + ) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/fyre/scaling.yaml b/benchmarks/fyre/scaling.yaml new file mode 100644 index 0000000..9712fd7 --- /dev/null +++ b/benchmarks/fyre/scaling.yaml @@ -0,0 +1,48 @@ +schema_version: 1 +benchmark_kind: comparison +infrastructure: + os: Ubuntu 24.04 + ssh_user: root + ssh_private_key: ~/.ssh/id_ed25519 + ssh_public_key: ~/.ssh/id_ed25519.pub + expiry_hours: 12 + helper_sizes: + - { cpu: 4, memory_gb: 16 } + - { cpu: 8, memory_gb: 32 } + - { cpu: 16, memory_gb: 32 } + initial_helpers: + locust: { cpu: 4, memory_gb: 16 } + fast_time: { cpu: 8, memory_gb: 32 } +images: + dataplane: ghcr.io/contextforge-org/contextforge-data-plane@sha256:0b3026b21659ce6c494142aee2c5866b44122232e73fcf21102da1f210692a96 + controlplane: ghcr.io/contextforge-org/cf-integration-fixture@sha256:5b206e6f863cea9f8cabea6451392428fbe67b16bdf10fefd1f7ac8032d95b52 + fast_time: ghcr.io/ibm/cfex-mcp-fast-time-server@sha256:110e1826f5d763e5afadba770b731dac93e0819c1bbadb68671b0124260603cf + helpers: ghcr.io/contextforge-org/cf-integration-helpers@sha256:f7e557e263737328225d827e73b519d0b40facb2698c55ee3d2de0ed82f00fa3 + locust: mirror.gcr.io/locustio/locust@sha256:fd39232c31971fe7509582717a95592f0fbf9a917b705f355bd7471e03bdd649 + postgres: postgres@sha256:4ef4dbc939d61acea57712655ddb4b4ab27419c913f94cca0cd57cb3ea3c2280 + redis: mirror.gcr.io/library/redis@sha256:a7859ed111db3c1f5404a973a4747505d559fb5ca32d37e447afc0ef845a2103 +workload: + protocol_version: 2026-07-28 + user_levels: [125, 250, 500, 1000] + first_users: 125 + maximum_users: 1000 + ramp_seconds: 30 + warmup_seconds: 30 + measure_seconds: 3600 + repetitions: 1 + maximum_campaign_seconds: 36000 + plateau_improvement_percent: 5.0 + boundary_percent: 12.5 + config_cache_seconds: 60 + helper_cpu_percent: 70.0 + helper_memory_percent: 80.0 + worker_core_percent: 85.0 + tools: + - convert_time + - echo + - get_stats + - get_system_time + - schema_success + - verify-protocol +scenarios: + - { id: comparison, label: Built-in dataplane vs external dataplane, replicas: 1, cpu: 4, memory_gb: 4, multiplier: 1 } diff --git a/benchmarks/fyre/terraform/.terraform.lock.hcl b/benchmarks/fyre/terraform/.terraform.lock.hcl new file mode 100644 index 0000000..65b19c2 --- /dev/null +++ b/benchmarks/fyre/terraform/.terraform.lock.hcl @@ -0,0 +1,15 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp-forge/fyre" { + version = "0.0.3" + constraints = "0.0.3" + hashes = [ + "h1:rSh9ZK9JGe8duTe3ygRL5mbJeOmKDdosb/hA4P/5hB4=", + "zh:39052c8ec42ebd4862d00187226e8967b78f5c71d6d7472a6281852ba10e9097", + "zh:53c4a6fd9d58afe0cb52773137bbf23067606af92157e88a36425985ce9931c1", + "zh:5a17a4060432477ff00cfd9474ebff8fd4219908739466238fed99a12069334b", + "zh:890df766e9b839623b1f0437355032a3c006226a6c200cd911e15ee1a9014e9f", + "zh:c06c24f7fc4e9c4b1c2b06af530f89e9d0ec94b13efa9adad3956f6a4900e153", + ] +} diff --git a/benchmarks/fyre/terraform/main.tf b/benchmarks/fyre/terraform/main.tf new file mode 100644 index 0000000..ef8f123 --- /dev/null +++ b/benchmarks/fyre/terraform/main.tf @@ -0,0 +1,111 @@ +data "fyre_user" "current" {} + +data "fyre_quota" "current" { + site = var.site +} + +locals { + account_default_product_group_id = try( + data.fyre_user.current.development.default_product_group_id == null + ? null + : tostring(data.fyre_user.current.development.default_product_group_id), + null, + ) + sole_product_group_id = try( + length(data.fyre_user.current.development.product_groups) == 1 + ? tostring(data.fyre_user.current.development.product_groups[0].id) + : null, + null, + ) + product_group_id = ( + var.product_group_id != null ? var.product_group_id : + local.account_default_product_group_id != null ? local.account_default_product_group_id : + local.sole_product_group_id + ) + quick_burn_available = contains( + ["true", "yes", "y"], + lower(try(data.fyre_user.current.development.quick_burn, "no")), + ) + common = { + os = var.os + platform = "x" + public_network = "y" + quota_type = local.product_group_id == null ? "quick_burn" : "product_group" + product_group_id = local.product_group_id + site = var.site + ssh_keys = [var.ssh_public_key] + } +} + +resource "fyre_vm" "locust" { + hostname = "cf-${var.run_id}-locust" + description = "ContextForge benchmark locust" + os = local.common.os + platform = local.common.platform + public_network = local.common.public_network + quota_type = local.common.quota_type + product_group_id = local.common.product_group_id + site = local.common.site + ssh_keys = local.common.ssh_keys + time_to_live = local.common.quota_type == "quick_burn" ? tostring(var.expiry_hours) : null + expiration = local.common.quota_type == "product_group" ? "${var.expiry_hours} hours" : null + cpu = var.locust_cpu + memory = var.locust_memory_gb + disable_delete = "n" + + lifecycle { + precondition { + condition = local.product_group_id != null || local.quick_burn_available + error_message = "FYRE account has no default or sole product group and no quick-burn quota; set FYRE_PRODUCT_GROUP_ID." + } + } +} + +resource "fyre_vm" "fast_time" { + hostname = "cf-${var.run_id}-fast-time" + description = "ContextForge benchmark fast time" + os = local.common.os + platform = local.common.platform + public_network = local.common.public_network + quota_type = local.common.quota_type + product_group_id = local.common.product_group_id + site = local.common.site + ssh_keys = local.common.ssh_keys + time_to_live = local.common.quota_type == "quick_burn" ? tostring(var.expiry_hours) : null + expiration = local.common.quota_type == "product_group" ? "${var.expiry_hours} hours" : null + cpu = var.fast_time_cpu + memory = var.fast_time_memory_gb + disable_delete = "n" + + lifecycle { + precondition { + condition = local.product_group_id != null || local.quick_burn_available + error_message = "FYRE account has no default or sole product group and no quick-burn quota; set FYRE_PRODUCT_GROUP_ID." + } + } +} + +resource "fyre_vm" "dataplane" { + count = var.dataplane_count + hostname = "cf-${var.run_id}-dataplane-${count.index + 1}" + description = "ContextForge benchmark dataplane replica ${count.index + 1}" + os = local.common.os + platform = local.common.platform + public_network = local.common.public_network + quota_type = local.common.quota_type + product_group_id = local.common.product_group_id + site = local.common.site + ssh_keys = local.common.ssh_keys + time_to_live = local.common.quota_type == "quick_burn" ? tostring(var.expiry_hours) : null + expiration = local.common.quota_type == "product_group" ? "${var.expiry_hours} hours" : null + cpu = var.dataplane_cpu + memory = var.dataplane_memory_gb + disable_delete = "n" + + lifecycle { + precondition { + condition = local.product_group_id != null || local.quick_burn_available + error_message = "FYRE account has no default or sole product group and no quick-burn quota; set FYRE_PRODUCT_GROUP_ID." + } + } +} diff --git a/benchmarks/fyre/terraform/outputs.tf b/benchmarks/fyre/terraform/outputs.tf new file mode 100644 index 0000000..4e5915f --- /dev/null +++ b/benchmarks/fyre/terraform/outputs.tf @@ -0,0 +1,52 @@ +locals { + locust_ips = { + for address in fyre_vm.locust.ips : address.type => address.ip + } + fast_time_ips = { + for address in fyre_vm.fast_time.ips : address.type => address.ip + } + dataplane_ips = [for vm in fyre_vm.dataplane : { + name = vm.hostname + id = vm.vm_id + ips = { for address in vm.ips : address.type => address.ip } + }] +} + +output "inventory" { + value = { + run_id = var.run_id + locust = { + name = fyre_vm.locust.hostname + id = fyre_vm.locust.vm_id + public_ip = try(local.locust_ips.public, "") + private_ip = try(local.locust_ips.private, "") + } + fast_time = { + name = fyre_vm.fast_time.hostname + id = fyre_vm.fast_time.vm_id + public_ip = try(local.fast_time_ips.public, "") + private_ip = try(local.fast_time_ips.private, "") + } + dataplanes = [for vm in local.dataplane_ips : { + name = vm.name + id = vm.id + public_ip = try(vm.ips.public, "") + private_ip = try(vm.ips.private, "") + }] + } +} + +output "quota" { + value = { + product_group_id = data.fyre_quota.current.details.product_group_id + product_group_name = data.fyre_quota.current.details.product_group_name + cpu = data.fyre_quota.current.details.x.cpu + cpu_used = data.fyre_quota.current.details.x.cpu_used + memory = data.fyre_quota.current.details.x.memory + memory_used = data.fyre_quota.current.details.x.memory_used + disk = data.fyre_quota.current.details.x.disk + disk_used = data.fyre_quota.current.details.x.disk_used + public_ips = data.fyre_quota.current.details.ip.public.quota + public_ips_used = data.fyre_quota.current.details.ip.public.used + } +} diff --git a/benchmarks/fyre/terraform/variables.tf b/benchmarks/fyre/terraform/variables.tf new file mode 100644 index 0000000..1024747 --- /dev/null +++ b/benchmarks/fyre/terraform/variables.tf @@ -0,0 +1,31 @@ +variable "run_id" { + type = string + validation { + condition = can(regex("^[a-z0-9](?:[a-z0-9-]{0,46}[a-z0-9])?$", var.run_id)) + error_message = "run_id must be a lowercase DNS-safe identifier of at most 48 characters." + } +} + +variable "os" { type = string } +variable "ssh_public_key" { type = string } +variable "expiry_hours" { type = number } +variable "dataplane_count" { type = number } +variable "dataplane_cpu" { type = number } +variable "dataplane_memory_gb" { type = number } +variable "locust_cpu" { type = number } +variable "locust_memory_gb" { type = number } +variable "fast_time_cpu" { type = number } +variable "fast_time_memory_gb" { type = number } + +variable "product_group_id" { + type = string + default = null + nullable = true + sensitive = true +} + +variable "site" { + type = string + default = null + nullable = true +} diff --git a/benchmarks/fyre/terraform/versions.tf b/benchmarks/fyre/terraform/versions.tf new file mode 100644 index 0000000..f282187 --- /dev/null +++ b/benchmarks/fyre/terraform/versions.tf @@ -0,0 +1,11 @@ +terraform { + required_version = ">= 1.8, < 2.0" + required_providers { + fyre = { + source = "hashicorp-forge/fyre" + version = "= 0.0.3" + } + } +} + +provider "fyre" {} diff --git a/benchmarks/fyre/test_campaign.py b/benchmarks/fyre/test_campaign.py new file mode 100644 index 0000000..92c88cf --- /dev/null +++ b/benchmarks/fyre/test_campaign.py @@ -0,0 +1,685 @@ +"""Unit coverage for FYRE capacity-search and report-input behavior.""" + +from __future__ import annotations + +import csv +import json +import sys +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +import campaign +import report + +sys.path.insert(0, str(Path(__file__).parent / "deploy")) +import monitor +import run_locust +import smoke + + +def passed(users: int, rps: float) -> dict: + return { + "passed": True, + "users": users, + "rps": rps, + "failures": 0, + "p50_ms": 1.0, + "p95_ms": 2.0, + "p99_ms": 3.0, + "per_replica_rps": {"MCP tools/call [replica-1]": rps}, + "pressure": {}, + } + + +def config() -> dict: + return { + "workload": { + "first_users": 125, + "maximum_users": 32_000, + "maximum_campaign_seconds": 21_600, + "plateau_improvement_percent": 5.0, + "boundary_percent": 12.5, + "repetitions": 3, + "warmup_seconds": 30, + "measure_seconds": 120, + "helper_cpu_percent": 70.0, + "helper_memory_percent": 80.0, + "worker_core_percent": 85.0, + }, + "images": {"locust": "locust@sha256:test"}, + } + + +class CapacityTests(unittest.TestCase): + @mock.patch.object(campaign, "reset_comparison_target") + @mock.patch.object(campaign, "smoke") + @mock.patch.object(campaign, "prepare_builtin_comparison") + @mock.patch.object(campaign, "prepare_rust_comparison") + @mock.patch.object(campaign, "one_phase") + def test_fixed_comparison_runs_the_eight_default_benchmarks( + self, phase, rust, builtin, _smoke, _reset + ): + rust.return_value = (["http://rust/mcp"], list(smoke.TOOLS), "rust.env") + builtin.return_value = ( + ["http://builtin/mcp"], + [f"fast_time_{name}" for name in smoke.TOOLS], + "builtin.env", + ) + phase.side_effect = lambda _r, _c, _i, _u, _o, users, *_args: passed( + users, float(users) + ) + test_config = config() + test_config.update( + { + "scenarios": [{"id": "comparison"}], + "workload": { + **test_config["workload"], + "protocol_version": "2026-07-28", + "user_levels": [125, 250, 500, 1000], + }, + } + ) + inventory = {"locust": {}, "fast_time": {}, "dataplanes": [{}]} + with tempfile.TemporaryDirectory() as directory: + result = campaign.fixed_comparison( + None, test_config, inventory, Path(directory) + ) + self.assertEqual(result["status"], "confirmed") + self.assertEqual( + [(item["lane"], item["users"]) for lane in result["runs"].values() for item in lane], + [ + ("rust", 125), + ("rust", 250), + ("rust", 500), + ("rust", 1000), + ("builtin", 125), + ("builtin", 250), + ("builtin", 500), + ("builtin", 1000), + ], + ) + + @mock.patch.object(campaign, "reset_comparison_target") + @mock.patch.object(campaign, "smoke") + @mock.patch.object(campaign, "prepare_builtin_comparison") + @mock.patch.object(campaign, "prepare_rust_comparison") + @mock.patch.object(campaign, "one_phase") + def test_fixed_comparison_stops_after_first_error( + self, phase, rust, builtin, _smoke, _reset + ): + rust.return_value = (["http://rust/mcp"], list(smoke.TOOLS), "rust.env") + phase.side_effect = [passed(125, 100.0), {"passed": False, "reason": "error"}] + test_config = config() + test_config.update( + { + "scenarios": [{"id": "comparison"}], + "workload": { + **test_config["workload"], + "protocol_version": "2026-07-28", + "user_levels": [125, 250, 500, 1000], + }, + } + ) + inventory = {"locust": {}, "fast_time": {}, "dataplanes": [{}]} + with tempfile.TemporaryDirectory() as directory: + result = campaign.fixed_comparison( + None, test_config, inventory, Path(directory) + ) + self.assertEqual(result["status"], "failed") + self.assertEqual(phase.call_count, 2) + builtin.assert_not_called() + + @mock.patch.object(campaign, "reset_comparison_target") + @mock.patch.object(campaign, "smoke") + @mock.patch.object(campaign, "prepare_builtin_comparison") + @mock.patch.object(campaign, "prepare_rust_comparison") + @mock.patch.object(campaign, "one_phase") + def test_fixed_comparison_requests_a_full_rerun_after_helper_saturation( + self, phase, rust, builtin, _smoke, _reset + ): + rust.return_value = (["http://rust/mcp"], list(smoke.TOOLS), "rust.env") + saturated = passed(125, 100.0) + saturated["pressure"] = {"locust": {"mean_cpu_percent": 71.0}} + phase.return_value = saturated + test_config = config() + test_config.update( + { + "scenarios": [{"id": "comparison"}], + "workload": { + **test_config["workload"], + "protocol_version": "2026-07-28", + "user_levels": [125, 250, 500, 1000], + }, + } + ) + inventory = {"locust": {}, "fast_time": {}, "dataplanes": [{}]} + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + with self.assertRaises(SystemExit) as exit_status: + campaign.fixed_comparison(None, test_config, inventory, root) + request = json.loads((root / "helper-request.json").read_text()) + result = json.loads((root / "result.json").read_text()) + self.assertEqual(exit_status.exception.code, campaign.HELPER_SATURATED) + self.assertEqual(request, {"role": "locust"}) + self.assertEqual(result["status"], "inconclusive") + builtin.assert_not_called() + + @mock.patch.object(campaign, "smoke") + @mock.patch.object(campaign, "one_phase") + @mock.patch.object(campaign, "measured_step") + def test_first_failure_never_advances_above_the_failed_load( + self, measured, phase, _smoke + ): + measured.side_effect = lambda _r, _c, _i, _u, _o, users, name: ( + passed(users, float(users)) + if name.startswith("confirm") or users <= 202 + else {"passed": False, "users": users, "reason": "first error"} + ) + phase.return_value = passed(202, 1000.0) + with tempfile.TemporaryDirectory() as directory: + result = campaign.capacity_search( + None, + config(), + { + "locust": {}, + "fast_time": {"private_ip": "10.0.0.2"}, + "dataplanes": [], + }, + [], + Path(directory), + ) + calls = [ + call.args[5] + for call in measured.call_args_list + if not call.args[6].startswith("confirm") + ] + first_failure = calls.index(250) + self.assertTrue(all(users <= 250 for users in calls[first_failure + 1 :])) + self.assertEqual(result["status"], "confirmed") + + @mock.patch.object(campaign, "smoke") + @mock.patch.object(campaign, "one_phase") + @mock.patch.object(campaign, "measured_step") + def test_two_sub_five_percent_steps_stop_at_plateau(self, measured, phase, _smoke): + search_rates = {125: 100.0, 250: 130.0, 500: 133.0, 1000: 134.0} + + def result(_r, _c, _i, _u, _o, users, name): + if name.startswith("search"): + return passed(users, search_rates[users]) + return passed(users, 132.0) + + measured.side_effect = result + phase.return_value = passed(281, 1000.0) + with tempfile.TemporaryDirectory() as directory: + result = campaign.capacity_search( + None, + config(), + { + "locust": {}, + "fast_time": {"private_ip": "10.0.0.2"}, + "dataplanes": [], + }, + [], + Path(directory), + ) + self.assertEqual(result["users"], 281) + self.assertEqual(result["plateau_boundary"]["below"]["users"], 250) + self.assertEqual(result["plateau_boundary"]["at_or_above"]["users"], 281) + self.assertLessEqual(result["plateau_boundary"]["width_percent"], 12.5) + calls = [(call.args[5], call.args[6]) for call in measured.call_args_list] + self.assertIn((375, "plateau-refine-375"), calls) + self.assertIn((312, "plateau-refine-312"), calls) + self.assertIn((281, "plateau-refine-281"), calls) + self.assertNotIn(2000, [users for users, _name in calls]) + + @mock.patch.object(campaign, "smoke") + @mock.patch.object(campaign, "one_phase") + @mock.patch.object(campaign, "measured_step") + def test_failed_candidate_confirmation_refines_and_confirms_a_lower_load( + self, measured, phase, _smoke + ): + search_rates = {125: 100.0, 250: 150.0, 500: 200.0} + test_config = config() + test_config["workload"]["maximum_users"] = 500 + + def result(_r, _c, _i, _u, _o, users, name): + if name == "confirm-1-500": + return {"passed": False, "users": users, "reason": "first error"} + if name.startswith("search"): + return passed(users, search_rates[users]) + if name == "confirm-refine-1-468": + return {"passed": False, "users": users, "reason": "first error"} + return passed(users, float(users)) + + measured.side_effect = result + phase.return_value = passed(437, 1000.0) + with tempfile.TemporaryDirectory() as directory: + result = campaign.capacity_search( + None, + test_config, + { + "locust": {}, + "fast_time": {"private_ip": "10.0.0.2"}, + "dataplanes": [], + }, + [], + Path(directory), + ) + + self.assertEqual(result["status"], "confirmed") + self.assertEqual(result["users"], 437) + self.assertEqual(len(result["confirmation_failures"]), 1) + calls = [(call.args[5], call.args[6]) for call in measured.call_args_list] + self.assertIn((375, "confirm-refine-1-375"), calls) + self.assertIn((437, "confirm-1-437"), calls) + + def test_helper_saturation_uses_sustained_thresholds(self): + result = {"pressure": {"locust": {"mean_cpu_percent": 71.0}}} + self.assertEqual(campaign.helper_saturation(config(), result), "locust") + result["pressure"]["locust"]["mean_cpu_percent"] = 69.0 + self.assertIsNone(campaign.helper_saturation(config(), result)) + + @mock.patch.object(campaign, "smoke") + @mock.patch.object(campaign, "one_phase") + def test_step_uses_one_continuous_warmup_and_measurement(self, phase, _smoke): + phase.return_value = passed(125, 100.0) + result = campaign.measured_step( + None, config(), {"locust": {}}, [], Path("unused"), 125, "step" + ) + self.assertTrue(result["passed"]) + phase.assert_called_once_with( + None, config(), {"locust": {}}, [], Path("unused"), 125, 120, "step" + ) + + def test_smoke_passes_script_once_to_python_entrypoint(self): + remote = mock.Mock() + campaign.smoke( + remote, + {"public_ip": "192.0.2.10"}, + ["http://192.0.2.20:4445/mcp"], + "locust@sha256:test", + ) + command = remote.ssh.call_args.args[1] + self.assertIn("--entrypoint python", command) + self.assertIn("--user 0:0", command) + self.assertIn("locust@sha256:test smoke.py --urls", command) + self.assertNotIn("locust@sha256:test python smoke.py", command) + + def test_smoke_uses_valid_convert_time_datetime(self): + self.assertEqual(smoke.TOOLS["convert_time"]["time"], "2025-06-21T16:00:00Z") + + def test_prepare_hosts_quotes_inventory_backend_url(self): + remote = mock.Mock() + remote.ssh.return_value = mock.Mock(stdout="test-token\n", returncode=0) + private_ip = "10.0.0.2; touch /tmp/unquoted" + test_config = { + "images": { + "dataplane": "dataplane@sha256:test", + "fast_time": "fast-time@sha256:test", + "helpers": "helpers@sha256:test", + "locust": "locust@sha256:test", + "redis": "redis@sha256:test", + }, + "workload": { + "config_cache_seconds": 60, + "protocol_version": "2026-07-28", + }, + } + inventory = { + "locust": {"public_ip": "192.0.2.10", "private_ip": "10.0.0.10"}, + "fast_time": {"public_ip": "192.0.2.20", "private_ip": private_ip}, + "dataplanes": [ + {"public_ip": "192.0.2.30", "private_ip": "10.0.0.30"} + ], + } + with ( + tempfile.TemporaryDirectory() as directory, + mock.patch.object(campaign, "bootstrap_hosts"), + mock.patch.object(campaign, "compose_up"), + mock.patch.object(campaign, "write_remote_file"), + ): + campaign.prepare_hosts( + test_config, + inventory, + remote, + Path("deploy"), + Path("bootstrap.yml"), + Path("known_hosts"), + Path(directory), + ) + command = next( + call.args[1] + for call in remote.ssh.call_args_list + if "config_writer fixture" in call.args[1] + ) + backend_url = f"http://{private_ip}:9080/mcp" + self.assertIn(campaign.shlex.quote(backend_url), command) + + def test_builtin_verify_protocol_alias_maps_to_fast_time_tool(self): + self.assertEqual( + smoke.base_tool_name("fast_time_verify_protocol"), "verify-protocol" + ) + + def test_compose_pull_retries_before_starting_containers(self): + remote = mock.Mock() + remote.ssh.side_effect = [ + mock.Mock(returncode=1), + mock.Mock(returncode=0), + mock.Mock(returncode=0), + ] + with mock.patch.object(campaign.time, "sleep") as sleep: + campaign.compose_up(remote, "192.0.2.10", "dataplane.compose.yaml") + self.assertEqual(remote.ssh.call_count, 3) + self.assertIn(" pull", remote.ssh.call_args_list[0].args[1]) + self.assertIn(" pull", remote.ssh.call_args_list[1].args[1]) + self.assertIn(" up -d --wait", remote.ssh.call_args_list[2].args[1]) + sleep.assert_called_once_with(5) + + def test_monitor_detaches_all_standard_streams_from_ssh(self): + remote = mock.Mock() + remote.ssh.return_value.stdout = "123\n" + self.assertEqual( + campaign.start_monitor(remote, "192.0.2.10", "locust", "phase-1"), 123 + ) + command = remote.ssh.call_args.args[1] + self.assertNotIn("cd ", command) + self.assertIn("cf-fyre/telemetry/phase-1.log 2>&1 & echo $!", command) + + @mock.patch.object(run_locust, "wait_for_cluster", return_value=0) + @mock.patch.object(run_locust, "container_state", return_value=("exited", 0)) + @mock.patch.object(run_locust, "docker") + def test_locust_containers_can_write_root_owned_reports( + self, docker, _state, _wait + ): + with tempfile.TemporaryDirectory() as directory: + args = [ + "run_locust.py", + "--image", + "locust@sha256:test", + "--users", + "1", + "--spawn-rate", + "1", + "--seconds", + "1", + "--workers", + "1", + "--output", + directory, + "--env-file", + "benchmark.secret.env", + "--reset-stats", + "--measurement-seconds", + "1", + "--warmup-seconds", + "1", + ] + with ( + mock.patch.object(sys, "argv", args), + mock.patch.object(run_locust.signal, "signal"), + mock.patch.object(run_locust.time, "sleep"), + mock.patch.object(run_locust, "cleanup"), + self.assertRaises(SystemExit) as exit_status, + ): + run_locust.main() + self.assertEqual(exit_status.exception.code, 0) + run_calls = [call for call in docker.call_args_list if call.args[0] == "run"] + self.assertEqual( + docker.call_args_list[0].args[:2], ("network", "create") + ) + for call in run_calls[:2]: + arguments = call.args + user_index = arguments.index("--user") + self.assertEqual(arguments[user_index + 1], "0:0") + master_arguments = run_calls[0].args + self.assertIn("MCP_WARMUP_SECONDS=1", master_arguments) + self.assertNotIn("--master-bind-host", master_arguments) + self.assertNotIn("--reset-stats", master_arguments) + worker_arguments = run_calls[1].args + master_index = worker_arguments.index("--master-host") + name_index = master_arguments.index("--name") + self.assertEqual( + worker_arguments[master_index + 1], master_arguments[name_index + 1] + ) + self.assertIn("MCP_REPLICA_OFFSET=0", worker_arguments) + + def test_pressure_excludes_ramp_and_warmup_samples(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "host.jsonl" + samples = [ + { + "kind": "sample", + "time": 100.0, + "cpu": { + "cpu": {"busy_percent": 100.0, "steal_percent": 30.0}, + "cpu0": {"busy_percent": 100.0, "steal_percent": 30.0}, + }, + "memory": {"used_percent": 99.0}, + "netstat": "TcpExt: ListenOverflows ListenDrops\nTcpExt: 0 0", + "docker_state": '{"Status":"running","OOMKilled":false}', + }, + { + "kind": "sample", + "time": 200.0, + "cpu": { + "cpu": {"busy_percent": 40.0, "steal_percent": 2.0}, + "cpu0": {"busy_percent": 45.0, "steal_percent": 2.0}, + }, + "memory": {"used_percent": 50.0}, + "netstat": "TcpExt: ListenOverflows ListenDrops\nTcpExt: 0 0", + "docker_state": '{"Status":"running","OOMKilled":false}', + }, + ] + path.write_text( + "".join(json.dumps(sample) + "\n" for sample in samples), + encoding="utf-8", + ) + result = campaign.pressure(path, after=150.0) + self.assertEqual(result["mean_cpu_percent"], 40.0) + self.assertEqual(result["max_memory_percent"], 50.0) + self.assertEqual(result["max_mean_core_percent"], 45.0) + self.assertEqual(result["mean_steal_percent"], 2.0) + self.assertFalse(result["worker_or_network_pressure"]) + + def test_pressure_detects_network_drops(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "host.jsonl" + samples = [] + for timestamp, drops in ((100.0, 0), (101.0, 1)): + samples.append( + { + "kind": "sample", + "time": timestamp, + "cpu": {}, + "memory": {"used_percent": 10.0}, + "netstat": ( + "TcpExt: ListenOverflows ListenDrops TCPBacklogDrop\n" + f"TcpExt: 0 {drops} 0" + ), + "docker_state": "", + } + ) + path.write_text( + "".join(json.dumps(sample) + "\n" for sample in samples), + encoding="utf-8", + ) + result = campaign.pressure(path) + self.assertTrue(result["worker_or_network_pressure"]) + + def test_docker_pressure_ignores_clean_exit_and_detects_oom(self): + self.assertFalse(campaign.docker_pressure("[]")) + self.assertFalse( + campaign.docker_pressure( + '{"Status":"exited","ExitCode":0,"OOMKilled":false}' + ) + ) + self.assertTrue( + campaign.docker_pressure( + '{"Status":"exited","ExitCode":137,"OOMKilled":true}' + ) + ) + self.assertTrue( + campaign.docker_pressure( + '[{"Status":"running","ExitCode":0,"OOMKilled":true}]' + ) + ) + + @mock.patch.object(run_locust.time, "sleep") + @mock.patch.object(run_locust, "docker") + @mock.patch.object(run_locust, "container_state") + def test_worker_exit_stops_the_master_immediately(self, state, docker, _sleep): + state.side_effect = [("running", 0), ("exited", 2)] + self.assertEqual(run_locust.wait_for_cluster("master", ["worker"]), 1) + docker.assert_called_once_with( + "stop", "--time", "1", "master", check=False, capture=True + ) + + @mock.patch.object(run_locust.time, "sleep") + @mock.patch.object(run_locust, "docker") + @mock.patch.object(run_locust, "container_state") + def test_clean_worker_exit_waits_for_clean_master(self, state, docker, _sleep): + state.side_effect = [("running", 0), ("exited", 0), ("exited", 0)] + self.assertEqual(run_locust.wait_for_cluster("master", ["worker"]), 0) + docker.assert_not_called() + + @mock.patch.object(run_locust.time, "sleep") + @mock.patch.object(run_locust, "docker") + @mock.patch.object(run_locust, "container_state") + def test_created_containers_are_allowed_to_finish_starting( + self, state, docker, _sleep + ): + state.side_effect = [ + ("created", 0), + ("running", 0), + ("created", 0), + ("running", 0), + ("running", 0), + ("exited", 0), + ] + self.assertEqual(run_locust.wait_for_cluster("master", ["worker"]), 0) + docker.assert_not_called() + + def test_stats_preserve_replica_rates_and_exclude_discovery(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "stats.csv" + fields = [ + "Type", + "Name", + "Request Count", + "Failure Count", + "Requests/s", + "50%", + "95%", + "99%", + ] + with path.open("w", newline="", encoding="utf-8") as stream: + writer = csv.DictWriter(stream, fieldnames=fields) + writer.writeheader() + writer.writerow( + { + "Name": "MCP server/discover", + "Request Count": 100, + "Failure Count": 0, + "Requests/s": 50, + "50%": 50, + "95%": 80, + "99%": 90, + } + ) + writer.writerow( + { + "Name": "MCP tools/call [replica-1]", + "Request Count": 1000, + "Failure Count": 0, + "Requests/s": 500, + "50%": 2, + "95%": 4, + "99%": 5, + } + ) + writer.writerow( + { + "Name": "MCP tools/call [replica-2]", + "Request Count": 900, + "Failure Count": 0, + "Requests/s": 450, + "50%": 3, + "95%": 5, + "99%": 7, + } + ) + writer.writerow( + { + "Name": "Aggregated", + "Request Count": 1900, + "Failure Count": 0, + "Requests/s": 950, + "50%": 2.5, + "95%": 4.5, + "99%": 6, + } + ) + result = campaign.read_stats(path) + aggregate = campaign.read_stats(path, use_aggregate=True) + self.assertEqual(result["requests"], 1900) + self.assertEqual(result["rps"], 950.0) + self.assertEqual(len(result["per_replica_rps"]), 2) + self.assertLess(result["p95_ms"], 5.0) + + self.assertEqual(aggregate["p50_ms"], 2.5) + self.assertEqual(aggregate["p95_ms"], 4.5) + self.assertEqual(aggregate["p99_ms"], 6.0) + + def test_comparison_report_writes_machine_readable_lane_results(self): + lane = { + "users": 125, + "requests": 1000, + "failures": 0, + "rps": 100.0, + "p50_ms": 10.0, + "p95_ms": 20.0, + "p99_ms": 30.0, + } + result = { + "status": "confirmed", + "runs": { + "builtin": [lane], + "rust": [{**lane, "requests": 2500, "rps": 250.0}], + }, + } + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + result_dir = root / "comparison" + result_dir.mkdir() + (result_dir / "result.json").write_text(json.dumps(result)) + report.comparison_report( + {"workload": {"user_levels": [125]}}, root, render=False + ) + summary = json.loads((root / "summary.json").read_text()) + with (root / "summary.csv").open(newline="") as stream: + csv_rows = list(csv.DictReader(stream)) + self.assertEqual(summary["rows"][0]["external_vs_built_in"], 2.5) + self.assertEqual(csv_rows[0]["external_dataplane_requests"], "2500") + + def test_monitor_calculates_cpu_and_memory_pressure(self): + cpu = monitor.cpu_percent( + {"cpu": [100, 0, 0, 900, 0, 0, 0, 0]}, + {"cpu": [150, 0, 0, 950, 0, 0, 0, 10]}, + ) + self.assertEqual(cpu["cpu"]["busy_percent"], 54.545) + self.assertEqual(cpu["cpu"]["steal_percent"], 9.091) + with mock.patch.object( + monitor, + "read", + return_value="MemTotal: 1000 kB\nMemAvailable: 250 kB\nSwapTotal: 100 kB\nSwapFree: 80 kB\n", + ): + memory = monitor.memory() + self.assertEqual(memory["used_percent"], 75.0) + self.assertEqual(memory["swap_free_kib"], 80) + + +if __name__ == "__main__": + unittest.main() diff --git a/benchmarks/fyre/vertical-low-memory.yaml b/benchmarks/fyre/vertical-low-memory.yaml new file mode 100644 index 0000000..93f4671 --- /dev/null +++ b/benchmarks/fyre/vertical-low-memory.yaml @@ -0,0 +1,46 @@ +schema_version: 1 +infrastructure: + os: Ubuntu 24.04 + ssh_user: root + ssh_private_key: ~/.ssh/id_ed25519 + ssh_public_key: ~/.ssh/id_ed25519.pub + expiry_hours: 8 + helper_sizes: + - { cpu: 2, memory_gb: 8 } + - { cpu: 4, memory_gb: 16 } + - { cpu: 8, memory_gb: 32 } + - { cpu: 16, memory_gb: 32 } + initial_helpers: + locust: { cpu: 4, memory_gb: 16 } + fast_time: { cpu: 8, memory_gb: 32 } +images: + dataplane: ghcr.io/contextforge-org/contextforge-data-plane@sha256:0b3026b21659ce6c494142aee2c5866b44122232e73fcf21102da1f210692a96 + fast_time: ghcr.io/ibm/cfex-mcp-fast-time-server@sha256:110e1826f5d763e5afadba770b731dac93e0819c1bbadb68671b0124260603cf + helpers: ghcr.io/contextforge-org/cf-integration-helpers@sha256:f7e557e263737328225d827e73b519d0b40facb2698c55ee3d2de0ed82f00fa3 + locust: mirror.gcr.io/locustio/locust@sha256:fd39232c31971fe7509582717a95592f0fbf9a917b705f355bd7471e03bdd649 + redis: mirror.gcr.io/library/redis@sha256:a7859ed111db3c1f5404a973a4747505d559fb5ca32d37e447afc0ef845a2103 +workload: + protocol_version: 2026-07-28 + first_users: 125 + maximum_users: 32000 + ramp_seconds: 30 + warmup_seconds: 30 + measure_seconds: 120 + repetitions: 3 + maximum_campaign_seconds: 21600 + plateau_improvement_percent: 5.0 + boundary_percent: 12.5 + config_cache_seconds: 60 + helper_cpu_percent: 70.0 + helper_memory_percent: 80.0 + worker_core_percent: 85.0 + tools: + - convert_time + - echo + - get_stats + - get_system_time + - schema_success + - verify-protocol +scenarios: + - { id: baseline, label: 2 vCPU / 2 GB, replicas: 1, cpu: 2, memory_gb: 2, multiplier: 1 } + - { id: vertical-2x, label: 4 vCPU / 4 GB, replicas: 1, cpu: 4, memory_gb: 4, multiplier: 2 } diff --git a/docker/docker-compose.cf-dataplane-config.yaml b/docker/docker-compose.cf-dataplane-config.yaml index 78fa9bf..d2358ae 100644 --- a/docker/docker-compose.cf-dataplane-config.yaml +++ b/docker/docker-compose.cf-dataplane-config.yaml @@ -1,4 +1,28 @@ services: + global_config_writer: + image: ${CF_HELPERS_IMAGE:-ghcr.io/contextforge-org/cf-integration-helpers:${CF_HARNESS_VERSION:?Set CF_HARNESS_VERSION to the cf-integration version}} + pull_policy: ${CF_HARNESS_PULL_POLICY:-missing} + build: + context: ${CF_INTEGRATION_ROOT:?Set CF_INTEGRATION_ROOT to the integration harness root} + dockerfile: docker/helpers.Dockerfile + labels: + name: cf-dataplane-global-config-writer + restart: "no" + networks: + - mcpnet + environment: + CF_CONFIG_REDIS_URL: redis://redis:6379 + command: + - __helper + - global-config + - --allowed-hosts + - ${CF_DATAPLANE_MCP_ALLOWED_HOSTS:-127.0.0.1:${NGINX_PORT:-8080},localhost:${NGINX_PORT:-8080},nginx:80} + - --allowed-origins + - ${CF_DATAPLANE_MCP_ALLOWED_ORIGINS:-http://127.0.0.1:${NGINX_PORT:-8080},http://localhost:${NGINX_PORT:-8080}} + depends_on: + redis: + condition: service_healthy + config_writer: profiles: ["helpers"] image: ${CF_HELPERS_IMAGE:-ghcr.io/contextforge-org/cf-integration-helpers:${CF_HARNESS_VERSION:?Set CF_HARNESS_VERSION to the cf-integration version}} diff --git a/docker/docker-compose.cf-dataplane-standalone.yaml b/docker/docker-compose.cf-dataplane-standalone.yaml index 3dcbd5f..2e42102 100644 --- a/docker/docker-compose.cf-dataplane-standalone.yaml +++ b/docker/docker-compose.cf-dataplane-standalone.yaml @@ -58,6 +58,11 @@ services: labels: name: cf-dataplane restart: "no" + cpuset: "${CF_LOAD_TARGET_CPUSET:-}" + ulimits: + nofile: + soft: 65536 + hard: 65536 networks: mcpnet: aliases: @@ -71,13 +76,12 @@ services: CONTEXTFORGE_DATA_PLANE_REDIS_CONNECTION_MODE: plain-text CONTEXTFORGE_DATA_PLANE_JWKS_URL: http://127.0.0.1:4446/.well-known/jwks.json CONTEXTFORGE_DATA_PLANE_UPSTREAM_CONNECTION_MODE: plain-text-or-tls - CONTEXTFORGE_GATEWAY_RS_MCP_ALLOWED_HOSTS: ${CF_DATAPLANE_MCP_ALLOWED_HOSTS:-127.0.0.1:${NGINX_PORT:-8080},localhost:${NGINX_PORT:-8080},nginx} - CONTEXTFORGE_GATEWAY_RS_MCP_ALLOWED_ORIGINS: ${CF_DATAPLANE_MCP_ALLOWED_ORIGINS:-http://127.0.0.1:${NGINX_PORT:-8080},http://localhost:${NGINX_PORT:-8080}} - CONTEXTFORGE_DATA_PLANE_USER_CONFIG_CACHE_EXPIRY_SECONDS: "0" RUST_LOG: ${CF_DATAPLANE_LOG:-info} depends_on: redis: condition: service_healthy + global_config_writer: + condition: service_completed_successfully nginx: image: nginx:1.30.4-alpine3.24 @@ -127,6 +131,11 @@ services: labels: name: cf-locust restart: "no" + cpuset: "${CF_LOAD_LOCUST_CPUSET:-}" + ulimits: + nofile: + soft: 65536 + hard: 65536 user: "${HOST_UID:-1000}:${HOST_GID:-1000}" networks: - mcpnet diff --git a/docker/docker-compose.cf-dataplane.yaml b/docker/docker-compose.cf-dataplane.yaml index 7e15294..0db9cfd 100644 --- a/docker/docker-compose.cf-dataplane.yaml +++ b/docker/docker-compose.cf-dataplane.yaml @@ -54,6 +54,10 @@ services: retries: 30 start_period: 2s + config_writer: + volumes: + - integration_auth:/keys:ro + dataplane: image: ${CF_DATAPLANE_IMAGE:?Set CF_DATAPLANE_IMAGE to the cf-dataplane image tag} pull_policy: ${CF_DATAPLANE_PULL_POLICY:-always} @@ -61,6 +65,11 @@ services: labels: name: cf-dataplane restart: unless-stopped + cpuset: "${CF_LOAD_TARGET_CPUSET:-}" + ulimits: + nofile: + soft: 65536 + hard: 65536 networks: mcpnet: aliases: @@ -77,19 +86,12 @@ services: CONTEXTFORGE_DATA_PLANE_REDIS_CONNECTION_MODE: plain-text CONTEXTFORGE_DATA_PLANE_JWKS_URL: http://127.0.0.1:4446/.well-known/jwks.json CONTEXTFORGE_DATA_PLANE_UPSTREAM_CONNECTION_MODE: plain-text-or-tls - # These two MCP transport settings intentionally retain the historical - # prefix in the current dataplane configuration contract. - CONTEXTFORGE_GATEWAY_RS_MCP_ALLOWED_HOSTS: ${CF_DATAPLANE_MCP_ALLOWED_HOSTS:-127.0.0.1:${NGINX_PORT:-8080},localhost:${NGINX_PORT:-8080},nginx} - CONTEXTFORGE_GATEWAY_RS_MCP_ALLOWED_ORIGINS: ${CF_DATAPLANE_MCP_ALLOWED_ORIGINS:-http://127.0.0.1:${NGINX_PORT:-8080},http://localhost:${NGINX_PORT:-8080}} - # Disable the per-subject config cache for functional runs: its sliding - # TTL freezes stale configs under steady traffic (retry loops renew it - # forever), hiding servers created after first contact. Set to 60 to - # restore the image default for load benchmarks. - CONTEXTFORGE_DATA_PLANE_USER_CONFIG_CACHE_EXPIRY_SECONDS: ${CF_DATAPLANE_USER_CONFIG_CACHE_EXPIRY_SECONDS:-0} RUST_LOG: ${CF_DATAPLANE_LOG:-info} depends_on: redis: condition: service_started + global_config_writer: + condition: service_completed_successfully deploy: replicas: 1 diff --git a/docker/docker-compose.cf-integration.yaml b/docker/docker-compose.cf-integration.yaml index fb09dc7..28dff0c 100644 --- a/docker/docker-compose.cf-integration.yaml +++ b/docker/docker-compose.cf-integration.yaml @@ -4,6 +4,11 @@ services: locust: + cpuset: "${CF_LOAD_LOCUST_CPUSET:-}" + ulimits: + nofile: + soft: 65536 + hard: 65536 volumes: # Harness locustfile with streamable-HTTP content negotiation; the # upstream locustfile_mcp_protocol.py sends Accept: application/json diff --git a/docker/docker-compose.cf-load-builtin.yaml b/docker/docker-compose.cf-load-builtin.yaml index d9be8c1..212263e 100644 --- a/docker/docker-compose.cf-load-builtin.yaml +++ b/docker/docker-compose.cf-load-builtin.yaml @@ -1,5 +1,15 @@ # Route builtin load directly to the active gateway listener with pooled connections. services: + gateway: + cpuset: "${CF_LOAD_TARGET_CPUSET:-}" + + locust: + cpuset: "${CF_LOAD_LOCUST_CPUSET:-}" + ulimits: + nofile: + soft: 65536 + hard: 65536 + nginx: volumes: - ${CF_INTEGRATION_ROOT:?Set CF_INTEGRATION_ROOT}/docker/nginx.cf-load-builtin.conf:/etc/nginx/nginx.conf:ro diff --git a/docker/helpers.Dockerfile b/docker/helpers.Dockerfile index 154f184..0e35752 100644 --- a/docker/helpers.Dockerfile +++ b/docker/helpers.Dockerfile @@ -4,6 +4,7 @@ COPY Cargo.toml Cargo.lock ./ COPY src ./src COPY docker ./docker COPY scripts ./scripts +COPY benchmarks ./benchmarks COPY tests/conformance/baselines ./tests/conformance/baselines RUN --mount=type=cache,target=/usr/local/cargo/registry \ --mount=type=cache,target=/usr/local/cargo/git \ diff --git a/docker/nginx.cf-dataplane-standalone.conf.template b/docker/nginx.cf-dataplane-standalone.conf.template index 11dbed8..a2f820c 100644 --- a/docker/nginx.cf-dataplane-standalone.conf.template +++ b/docker/nginx.cf-dataplane-standalone.conf.template @@ -7,12 +7,17 @@ upstream dataplane_backend { keepalive_timeout 60s; } +map $http_host $dataplane_host { + default $http_host; + "nginx" "nginx:80"; +} + server { listen 80; server_name localhost; proxy_http_version 1.1; - proxy_set_header Host $http_host; + proxy_set_header Host $dataplane_host; proxy_set_header Authorization $http_authorization; proxy_set_header Mcp-Session-Id $http_mcp_session_id; proxy_set_header Mcp-Protocol-Version $http_mcp_protocol_version; diff --git a/docker/nginx.cf-dataplane.conf b/docker/nginx.cf-dataplane.conf index d505a02..b08e87f 100644 --- a/docker/nginx.cf-dataplane.conf +++ b/docker/nginx.cf-dataplane.conf @@ -56,6 +56,11 @@ http { "" $http_host; } + map $http_host $dataplane_host { + default $http_host; + "nginx" "nginx:80"; + } + server { listen 80 backlog=4096 reuseport; server_name localhost; @@ -64,7 +69,7 @@ http { # while pooling connections across requests. proxy_http_version 1.1; - proxy_set_header Host $http_host; + proxy_set_header Host $dataplane_host; proxy_set_header Authorization $http_authorization; proxy_set_header Mcp-Session-Id $http_mcp_session_id; proxy_set_header Mcp-Protocol-Version $http_mcp_protocol_version; diff --git a/scripts/locustfile_mcp.py b/scripts/locustfile_mcp.py index 17e305c..f8f4ec2 100644 --- a/scripts/locustfile_mcp.py +++ b/scripts/locustfile_mcp.py @@ -12,29 +12,52 @@ MCPGATEWAY_BEARER_TOKEN bearer token (required) MCP_TOOL_NAMES optional comma-separated tools to call MCP_SKIP_TOOL_LIST true when direct tool aliases are supplied + MCP_BASE_URLS optional comma-separated replica origins + MCP_REPLICA_OFFSET worker-specific replica rotation offset + MCP_DIRECT_DATAPLANE use the native dataplane route without nginx + MCP_FYRE_WORKLOAD enable the six-tool FYRE workload arguments + MCP_EXPLICIT_ZERO_DELAY send zero delay to Fast Time echo + MCP_MEASUREMENT_MARKER FYRE path written after ramp and warmup + MCP_MEASUREMENT_SECONDS FYRE measured duration after the marker + MCP_WARMUP_SECONDS FYRE steady-state warmup after spawning LOCUST_REQUEST_TIMEOUT_SECONDS positive finite per-request timeout (default 60) """ + from __future__ import annotations +import itertools import json +import logging import math import os import random +import time import uuid +from pathlib import Path from urllib.parse import quote import gevent -from locust import HttpUser, between, events, task +from locust import constant, events, task + +try: + from locust import FastHttpUser +except ImportError: # Minimal test doubles expose only HttpUser. + from locust import HttpUser as FastHttpUser +from locust.runners import MasterRunner, WorkerRunner PROTOCOL_VERSION = os.environ.get("MCP_PROTOCOL_VERSION", "2026-07-28") STATELESS = PROTOCOL_VERSION == "2026-07-28" LEGACY_PROTOCOL_VERSIONS = {"2024-11-05", "2025-03-26", "2025-06-18", "2025-11-25"} if PROTOCOL_VERSION not in {"2025-11-25", "2026-07-28"}: - raise RuntimeError("MCP_PROTOCOL_VERSION must be a harness-selected client revision") + raise RuntimeError( + "MCP_PROTOCOL_VERSION must be a harness-selected client revision" + ) ACCEPT = "application/json, text/event-stream" _REQUEST_TIMEOUT_ERROR = ( "LOCUST_REQUEST_TIMEOUT_SECONDS must be a finite number greater than zero" ) +_FAIL_FAST_MESSAGE = "cf_integration_fail_fast" +_LOGGER = logging.getLogger(__name__) def _request_timeout_seconds() -> float: @@ -54,6 +77,17 @@ def _request_timeout_seconds() -> float: "fast_time_echo": {"message": "cf-integration"}, "fast-time-echo": {"message": "cf-integration"}, } +_FYRE_TOOL_ARGUMENTS = { + "convert_time": { + "time": "2025-06-21T16:00:00Z", + "source_timezone": "UTC", + "target_timezone": "Europe/Dublin", + }, + "get_stats": {}, + "get_system_time": {"timezone": "UTC"}, + "schema_success": {}, + "verify-protocol": {}, +} def jsonrpc(method: str, params: dict | None = None) -> dict: @@ -120,7 +154,27 @@ def parse_mcp_body(text: str, content_type: str): def tool_call_args(tool_name: str) -> dict | None: """Use the same Fast Time echo payload for raw and control-plane aliases.""" arguments = _TOOL_ARGUMENTS.get(tool_name) - return dict(arguments) if arguments is not None else None + if ( + arguments is None + and os.environ.get("MCP_FYRE_WORKLOAD", "false").lower() == "true" + ): + base_name = tool_name + for prefix in ("fast_time_", "fast-time-"): + if base_name.startswith(prefix): + base_name = base_name[len(prefix) :] + break + if base_name == "verify_protocol": + base_name = "verify-protocol" + arguments = _TOOL_ARGUMENTS.get(base_name) or _FYRE_TOOL_ARGUMENTS.get(base_name) + if arguments is None: + return None + result = dict(arguments) + if ( + tool_name in {"echo", "fast_time_echo", "fast-time-echo"} + and os.environ.get("MCP_EXPLICIT_ZERO_DELAY", "false").lower() == "true" + ): + result["delay"] = 0 + return result def validate_result(method: str, result) -> dict: @@ -130,7 +184,9 @@ def validate_result(method: str, result) -> dict: if method == "initialize": version = result.get("protocolVersion") if not isinstance(version, str) or version not in LEGACY_PROTOCOL_VERSIONS: - raise ValueError("initialize must negotiate a supported legacy protocol revision") + raise ValueError( + "initialize must negotiate a supported legacy protocol revision" + ) if not isinstance(result.get("capabilities"), dict): raise ValueError("initialize result must include capabilities") server_info = result.get("serverInfo") @@ -138,11 +194,15 @@ def validate_result(method: str, result) -> dict: isinstance(server_info.get(field), str) and server_info[field] for field in ("name", "version") ): - raise ValueError("initialize result must include serverInfo name and version") + raise ValueError( + "initialize result must include serverInfo name and version" + ) elif method == "server/discover": versions = result.get("supportedVersions") if not isinstance(versions, list) or PROTOCOL_VERSION not in versions: - raise ValueError("server/discover must advertise the requested protocol version") + raise ValueError( + "server/discover must advertise the requested protocol version" + ) if not isinstance(result.get("capabilities"), dict): raise ValueError("server/discover result must include capabilities") if not isinstance(result.get("resultType"), str): @@ -180,11 +240,29 @@ def validate_result(method: str, result) -> dict: raise ValueError("tools/call result contains invalid content") return result + MCP_SERVER_ID = os.environ.get("MCP_SERVER_ID", "") MCP_STACK_MODE = os.environ.get("MCP_STACK_MODE", "dataplane") BEARER_TOKEN = os.environ.get("MCPGATEWAY_BEARER_TOKEN", "") -TOOL_NAMES = [name.strip() for name in os.environ.get("MCP_TOOL_NAMES", "").split(",") if name.strip()] +TOOL_NAMES = [ + name.strip() + for name in os.environ.get("MCP_TOOL_NAMES", "").split(",") + if name.strip() +] SKIP_TOOL_LIST = os.environ.get("MCP_SKIP_TOOL_LIST", "false").lower() == "true" +BASE_URLS = [ + url.strip().rstrip("/") + for url in os.environ.get("MCP_BASE_URLS", "").split(",") + if url.strip() +] +DIRECT_DATAPLANE = os.environ.get("MCP_DIRECT_DATAPLANE", "false").lower() == "true" +try: + _REPLICA_OFFSET = int(os.environ.get("MCP_REPLICA_OFFSET", "0")) +except ValueError: + raise RuntimeError("MCP_REPLICA_OFFSET must be a non-negative integer") from None +if _REPLICA_OFFSET < 0: + raise RuntimeError("MCP_REPLICA_OFFSET must be a non-negative integer") +_TARGET_SEQUENCE = itertools.count(_REPLICA_OFFSET) def safe_diagnostic(value) -> str: @@ -197,6 +275,8 @@ def mcp_path() -> str: """Return the mode-aware public MCP route.""" if MCP_STACK_MODE == "controlplane": return "/mcp" + if DIRECT_DATAPLANE: + return f"/contextforge-rs/servers/{quote(MCP_SERVER_ID, safe='')}/mcp" return f"/servers/{quote(MCP_SERVER_ID, safe='')}/mcp" @@ -205,13 +285,62 @@ def install_fail_fast(environment, **_kwargs) -> None: """Stop after the first request or user error, retaining failed-run reports.""" stopping = False + def stop_runner(): + nonlocal stopping + if stopping: + return + stopping = True + environment.process_exit_code = 1 + # Let the triggering event finish recording statistics before stopping users. + gevent.spawn_later(0, environment.runner.quit) + + def stop_from_worker(msg=None, **_message): + data = getattr(msg, "data", None) + detail = data.get("error") if isinstance(data, dict) else None + worker = data.get("worker") if isinstance(data, dict) else None + _LOGGER.error( + "Distributed worker %s failed: %s", + worker or "", + detail or "unspecified error", + ) + stop_runner() + + if isinstance(environment.runner, MasterRunner): + environment.runner.register_message(_FAIL_FAST_MESSAGE, stop_from_worker) + + marker = os.environ.get("MCP_MEASUREMENT_MARKER") + if marker: + warmup_seconds = float(os.environ["MCP_WARMUP_SECONDS"]) + measurement_seconds = float(os.environ["MCP_MEASUREMENT_SECONDS"]) + + def begin_measurement() -> None: + environment.runner.stats.reset_all() + if isinstance(environment.runner, MasterRunner): + Path(marker).write_text(f"{time.time()}\n", encoding="utf-8") + gevent.spawn_later(measurement_seconds, environment.runner.quit) + + def finish_warmup(**_kwargs) -> None: + gevent.spawn_later(warmup_seconds, begin_measurement) + + environment.events.spawning_complete.add_listener(finish_warmup) + def stop_on_error(exception=None, **_kwargs): nonlocal stopping if exception is not None and not stopping: stopping = True environment.process_exit_code = 1 - # Let the request event finish recording statistics before stopping users. - gevent.spawn_later(0, environment.runner.quit) + if isinstance(environment.runner, WorkerRunner): + gevent.spawn_later( + 0, + environment.runner.send_message, + _FAIL_FAST_MESSAGE, + { + "error": safe_diagnostic(exception), + "worker": getattr(environment.runner, "client_id", ""), + }, + ) + else: + gevent.spawn_later(0, environment.runner.quit) environment.events.request.add_listener(stop_on_error) environment.events.user_error.add_listener(stop_on_error) @@ -220,16 +349,25 @@ def stop_on_error(exception=None, **_kwargs): @events.quitting.add_listener def fail_empty_run(environment, **_kwargs) -> None: """Fail closed when user setup prevented every request.""" - if environment.stats.total.num_requests == 0: + if ( + not isinstance(environment.runner, WorkerRunner) + and environment.stats.total.num_requests == 0 + ): environment.process_exit_code = 1 -class MCPGatewayUser(HttpUser): +class MCPGatewayUser(FastHttpUser): """Drives discovery or initialization, then tool requests on the public route.""" - wait_time = between(0.05, 0.2) + wait_time = constant(0) + host = BASE_URLS[0] if BASE_URLS else None def __init__(self, *args, **kwargs): + self._replica_index = ( + next(_TARGET_SEQUENCE) % len(BASE_URLS) if BASE_URLS else None + ) + if self._replica_index is not None: + self.host = BASE_URLS[self._replica_index] super().__init__(*args, **kwargs) self._session_id: str | None = None self._protocol_version = PROTOCOL_VERSION @@ -268,7 +406,9 @@ def on_start(self): raise RuntimeError("initialize response did not include Mcp-Session-Id") if not STATELESS: self._protocol_version = result["protocolVersion"] - if not self._mcp_notification("notifications/initialized", None, name="MCP initialized"): + if not self._mcp_notification( + "notifications/initialized", None, name="MCP initialized" + ): return if not self._tool_names and not SKIP_TOOL_LIST: listed = self._mcp_request("tools/list", {}, name="MCP tools/list") @@ -280,9 +420,13 @@ def on_start(self): and isinstance(tool.get("name"), str) and tool["name"].strip() ] - self._tool_names = [name for name in self._tool_names if tool_call_args(name) is not None] + self._tool_names = [ + name for name in self._tool_names if tool_call_args(name) is not None + ] if not self._tool_names: - raise RuntimeError("Fast Time echo tool is required; refusing an empty load workload") + raise RuntimeError( + "Fast Time echo tool is required; refusing an empty load workload" + ) self._ready = True def on_stop(self): @@ -299,7 +443,9 @@ def on_stop(self): if not self._validate_backend(response): return if response.status_code not in (200, 202, 204, 404, 405): - response.failure(f"HTTP {response.status_code}; expected session termination response") + response.failure( + f"HTTP {response.status_code}; expected session termination response" + ) return response.success() @@ -339,9 +485,13 @@ def _headers( @staticmethod def _validate_backend(response) -> bool: - if MCP_STACK_MODE != "dataplane": + if MCP_STACK_MODE != "dataplane" or DIRECT_DATAPLANE: return True - marker = response.headers.get("X-CF-Integration-Backend") if response.headers else None + marker = ( + response.headers.get("X-CF-Integration-Backend") + if response.headers + else None + ) if marker != "dataplane": response.failure("Missing or invalid dataplane backend marker") return False @@ -373,16 +523,25 @@ def _mcp_request( ) as response: if not self._validate_backend(response): return None - session_id = response.headers.get("Mcp-Session-Id") if response.headers else None + session_id = ( + response.headers.get("Mcp-Session-Id") if response.headers else None + ) if session_id and not STATELESS: self._session_id = session_id if response.status_code != 200: detail = getattr(response, "error", None) - response.failure(safe_diagnostic(f"HTTP {response.status_code}" + (f": {detail}" if detail else ""))) + response.failure( + safe_diagnostic( + f"HTTP {response.status_code}" + + (f": {detail}" if detail else "") + ) + ) return None try: - message = parse_mcp_body(response.text, response.headers.get("Content-Type", "")) + message = parse_mcp_body( + response.text, response.headers.get("Content-Type", "") + ) except ValueError as exc: response.failure(safe_diagnostic(f"Invalid body: {exc}")) return None @@ -428,7 +587,12 @@ def _mcp_notification(self, method: str, params: dict | None, name: str) -> bool return False if response.status_code != 202: detail = getattr(response, "error", None) - response.failure(safe_diagnostic(f"HTTP {response.status_code}; expected 202" + (f": {detail}" if detail else ""))) + response.failure( + safe_diagnostic( + f"HTTP {response.status_code}; expected 202" + + (f": {detail}" if detail else "") + ) + ) return False if response.content: response.failure("HTTP 202 notification response body must be empty") @@ -442,4 +606,7 @@ def tools_call(self): return tool = random.choice(self._tool_names) args = tool_call_args(tool) - self._mcp_request("tools/call", {"name": tool, "arguments": args}, name="MCP tools/call") + name = "MCP tools/call" + if self._replica_index is not None: + name += f" [replica-{self._replica_index + 1}]" + self._mcp_request("tools/call", {"name": tool, "arguments": args}, name=name) diff --git a/src/app.rs b/src/app.rs index a46221a..bcb3d4d 100644 --- a/src/app.rs +++ b/src/app.rs @@ -15,7 +15,7 @@ use crate::performance::LoadRequest; use anyhow::{Result, bail}; use crate::cli::{ - CiCommand, Cli, CliLane, CliRoutedLane, Command, ConformanceCommand, DebugCommand, + CiCommand, Cli, CliLane, CliRoutedLane, Command, ConformanceCommand, DebugCommand, FyreCommand, LaneSelection, LiveGroup, LoadCommand, ProtocolVersion, StackCommand, TokenKind, }; const LANE_ENV: &str = "CF_MCP_LANE"; @@ -31,6 +31,7 @@ pub(crate) enum Action { protocol_version: ProtocolVersion, }, Load(ResolvedLoadArgs), + Fyre(FyreAction), Live { lane: SemanticLane, group: LiveGroup, @@ -53,6 +54,9 @@ impl Action { Self::Stack(StackAction::Config { .. }) => "stack config", Self::Probe { .. } => "probe", Self::Load(_) => "load test", + Self::Fyre(FyreAction::Run { .. }) => "FYRE benchmark campaign", + Self::Fyre(FyreAction::Status { .. }) => "FYRE benchmark status", + Self::Fyre(FyreAction::Destroy { .. }) => "FYRE benchmark destroy", Self::Live { .. } => "live tests", Self::Conformance(ConformanceAction::Run { .. }) => "conformance tests", Self::Conformance(ConformanceAction::Report { .. }) => "conformance report", @@ -93,8 +97,15 @@ impl Action { if args.observability { summary.push_str("\nObservability: ClickStack enabled during load"); } + if let Some(limit) = &args.builtin_memory_limit { + summary.push_str(&format!("\nBuilt-in gateway memory limit: {limit}")); + } + if args.isolate_cpus { + summary.push_str("\nCPU isolation: target and Locust split evenly"); + } summary } + Self::Fyre(action) => action.startup_summary(), Self::Live { lane, protocol_version, @@ -149,6 +160,7 @@ impl Action { self, Self::Stack(StackAction::Up { .. }) | Self::Load(_) + | Self::Fyre(FyreAction::Run { .. }) | Self::Conformance(ConformanceAction::Run { .. }) ) } @@ -174,6 +186,7 @@ impl Action { | Self::Stack(StackAction::Logs { standalone, .. }) | Self::Stack(StackAction::Config { standalone, .. }) => *standalone, Self::Load(args) => args.standalone, + Self::Fyre(_) => true, Self::Live { .. } => false, }; if standalone { @@ -184,6 +197,32 @@ impl Action { } } +/// A resolved operation on one FYRE benchmark run. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum FyreAction { + Run { + file: Option, + run_id: Option, + }, + Status { + run_id: String, + }, + Destroy { + run_id: String, + }, +} + +impl FyreAction { + fn startup_summary(&self) -> String { + let (operation, run_id) = match self { + Self::Run { run_id, .. } => ("run", run_id.as_deref().unwrap_or("generated")), + Self::Status { run_id, .. } => ("status", run_id.as_str()), + Self::Destroy { run_id, .. } => ("destroy", run_id.as_str()), + }; + format!("Infrastructure: FYRE\nOperation: {operation}\nRun ID: {run_id}") + } +} + impl StackAction { fn startup_summary(&self) -> String { let lane = match self { @@ -310,6 +349,8 @@ pub(crate) struct ResolvedLoadArgs { pub(crate) client_era: ProtocolVersion, pub(crate) standalone: bool, pub(crate) observability: bool, + pub(crate) builtin_memory_limit: Option, + pub(crate) isolate_cpus: bool, pub(crate) request: LoadRequest, } @@ -392,23 +433,49 @@ pub(crate) fn resolve_action(cli: Cli, environment: &Environment) -> Result { - let LoadCommand::Run(args) = args.command; - let topology = resolve_lane(args.lane, environment)?; - validate_standalone_lane(standalone, topology)?; - Ok(Action::Load(ResolvedLoadArgs { - topology, - client_era: args.client_era, - standalone, - observability: args.observability, - request: LoadRequest { - smoke: args.smoke, - users: args.users, - spawn_rate: args.spawn_rate, - run_time: args.run_time, - }, - })) - } + Command::Load(args) => match args.command { + LoadCommand::Run(args) => { + let topology = resolve_lane(args.lane, environment)?; + validate_standalone_lane(standalone, topology)?; + if args.builtin_memory_limit.is_some() && topology != StackMode::Controlplane { + bail!("--builtin-memory-limit requires --lane builtin"); + } + Ok(Action::Load(ResolvedLoadArgs { + topology, + client_era: args.client_era, + standalone, + observability: args.observability, + builtin_memory_limit: args.builtin_memory_limit, + isolate_cpus: args.isolate_cpus, + request: LoadRequest { + smoke: args.smoke, + users: args.users, + spawn_rate: args.spawn_rate, + run_time: args.run_time, + workers: args.workers, + }, + })) + } + LoadCommand::Fyre(args) => { + if standalone { + bail!( + "--standalone is not used by load fyre; FYRE targets are always isolated" + ); + } + Ok(Action::Fyre(match args.command { + FyreCommand::Run(args) => FyreAction::Run { + file: args.file, + run_id: args.run_id, + }, + FyreCommand::Status(args) => FyreAction::Status { + run_id: validated_run_id(args.run_id)?, + }, + FyreCommand::Destroy(args) => FyreAction::Destroy { + run_id: validated_run_id(args.run_id)?, + }, + })) + } + }, Command::Live(args) => { let lane = resolve_live_lane(args.target.lane, environment)?; if standalone { @@ -525,6 +592,30 @@ pub(crate) fn resolve_action(cli: Cli, environment: &Environment) -> Result Result { + if is_valid_run_id(&run_id) { + Ok(run_id) + } else { + bail!("--run-id must contain only lowercase letters, digits, and hyphens") + } +} + +fn is_valid_run_id(run_id: &str) -> bool { + !run_id.is_empty() + && run_id.len() <= 48 + && run_id + .bytes() + .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'-') + && run_id + .as_bytes() + .first() + .is_some_and(u8::is_ascii_alphanumeric) + && run_id + .as_bytes() + .last() + .is_some_and(u8::is_ascii_alphanumeric) +} + fn environment_utf8(environment: &Environment, key: &str) -> Option { environment .get(std::ffi::OsStr::new(key)) diff --git a/src/app_tests.rs b/src/app_tests.rs index 520aa27..4afd099 100644 --- a/src/app_tests.rs +++ b/src/app_tests.rs @@ -2,7 +2,8 @@ use std::ffi::OsString; use std::path::PathBuf; use cf_integration::app::{ - Action, CiAction, ConformanceAction, DebugAction, ResolvedLoadArgs, StackAction, resolve_action, + Action, CiAction, ConformanceAction, DebugAction, FyreAction, ResolvedLoadArgs, StackAction, + resolve_action, }; use cf_integration::cli::{Cli, LaneSelection, LiveGroup, ProtocolVersion, TokenKind}; use cf_integration::conformance::results::{ConformanceServerEra, SemanticLane}; @@ -30,6 +31,32 @@ fn every_subcommand_has_a_stable_progress_description() { (&["cf-integration", "stack", "config"], "stack config"), (&["cf-integration", "probe"], "probe"), (&["cf-integration", "load", "run"], "load test"), + ( + &["cf-integration", "load", "fyre", "run"], + "FYRE benchmark campaign", + ), + ( + &[ + "cf-integration", + "load", + "fyre", + "status", + "--run-id", + "scale-run", + ], + "FYRE benchmark status", + ), + ( + &[ + "cf-integration", + "load", + "fyre", + "destroy", + "--run-id", + "scale-run", + ], + "FYRE benchmark destroy", + ), (&["cf-integration", "live"], "live tests"), ( &["cf-integration", "conformance", "run"], @@ -201,11 +228,54 @@ fn conformance_startup_labels_both_legacy_era_selections() { fn multi_phase_commands_own_detailed_progress_while_simple_commands_use_global_progress() { assert!(!action(&["cf-integration", "stack", "up"], &[]).uses_global_activity()); assert!(!action(&["cf-integration", "load", "run"], &[]).uses_global_activity()); + assert!(!action(&["cf-integration", "load", "fyre", "run"], &[]).uses_global_activity()); assert!(!action(&["cf-integration", "conformance", "run"], &[]).uses_global_activity()); assert!(action(&["cf-integration", "stack", "down"], &[]).uses_global_activity()); assert!(action(&["cf-integration", "probe"], &[]).uses_global_activity()); } +#[test] +fn fyre_actions_are_isolated_runtime_operations() { + assert_eq!( + action( + &[ + "cf-integration", + "load", + "fyre", + "run", + "--file", + "matrix.yaml", + "--run-id", + "scale-run", + ], + &[], + ), + Action::Fyre(FyreAction::Run { + file: Some(PathBuf::from("matrix.yaml")), + run_id: Some("scale-run".to_owned()), + }) + ); + let status = action( + &[ + "cf-integration", + "load", + "fyre", + "status", + "--run-id", + "scale-run", + ], + &[], + ); + assert_eq!( + status.config_requirements(), + ConfigRequirements::StandaloneRuntime + ); + assert_eq!( + status.startup_summary(), + "Infrastructure: FYRE\nOperation: status\nRun ID: scale-run" + ); +} + #[test] fn lane_precedence_is_cli_then_environment_then_external() { assert_eq!( @@ -350,6 +420,11 @@ fn load_preserves_explicit_locust_settings() { "0.5", "--run-time", "10s", + "--workers", + "4", + "--builtin-memory-limit", + "16G", + "--isolate-cpus", ], &[], ), @@ -358,11 +433,14 @@ fn load_preserves_explicit_locust_settings() { client_era: ProtocolVersion::Legacy, standalone: false, observability: false, + builtin_memory_limit: Some("16G".to_owned()), + isolate_cpus: true, request: LoadRequest { smoke: true, users: Some(2), spawn_rate: Some(0.5), run_time: Some("10s".to_owned()), + workers: Some(4), }, }) ); @@ -388,11 +466,14 @@ fn standalone_load_is_external_only() { client_era: ProtocolVersion::default(), standalone: true, observability: false, + builtin_memory_limit: None, + isolate_cpus: false, request: LoadRequest { smoke: false, users: None, spawn_rate: None, run_time: None, + workers: None, }, }) ); @@ -413,6 +494,23 @@ fn standalone_load_is_external_only() { let error = resolve_action(cli, &Environment::new()) .expect_err("standalone mode must reject the built-in lane"); assert_eq!(error.to_string(), "--standalone requires --lane external"); + + let cli = Cli::try_parse_from([ + "cf-integration", + "load", + "run", + "--lane", + "external", + "--builtin-memory-limit", + "16G", + ]) + .expect("CLI syntax should parse before lane validation"); + let error = resolve_action(cli, &Environment::new()) + .expect_err("the built-in memory limit must reject the external lane"); + assert_eq!( + error.to_string(), + "--builtin-memory-limit requires --lane builtin" + ); } #[test] diff --git a/src/cli.rs b/src/cli.rs index 081d47c..4ba9bed 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -76,6 +76,20 @@ fn parse_run_time(value: &str) -> Result { Ok(value.to_owned()) } +fn parse_memory_limit(value: &str) -> Result { + let digits = value.bytes().take_while(u8::is_ascii_digit).count(); + let (amount, unit) = value.split_at(digits); + let valid_amount = amount.parse::().is_ok_and(|amount| amount > 0); + let valid_unit = matches!(unit.to_ascii_lowercase().as_str(), "b" | "k" | "m" | "g"); + if valid_amount && valid_unit { + Ok(value.to_owned()) + } else { + Err(String::from( + "must be a positive Docker memory limit such as 16G", + )) + } +} + /// Orchestrates built-in and external dataplane integration workflows. #[derive(Debug, Clone, PartialEq, Parser)] #[command(name = "cf-integration", version, arg_required_else_help = true)] @@ -309,6 +323,51 @@ pub(crate) enum LoadCommand { /// Run Locust through the selected public MCP route. #[command(visible_alias = "r")] Run(LoadRunArgs), + /// Run repeatable comparison and scaling benchmarks on FYRE VMs. + #[command(visible_alias = "f")] + Fyre(FyreArgs), +} + +/// FYRE benchmark command selection. +#[derive(Debug, Clone, PartialEq, Eq, Args)] +pub(crate) struct FyreArgs { + /// FYRE benchmark operation to run. + #[command(subcommand)] + pub(crate) command: FyreCommand, +} + +/// Operations on one FYRE benchmark run. +#[derive(Debug, Clone, PartialEq, Eq, Subcommand)] +pub(crate) enum FyreCommand { + /// Provision, benchmark, download reports, and destroy run-owned VMs. + #[command(visible_alias = "r")] + Run(FyreRunArgs), + /// Show durable state for a benchmark run. + #[command(visible_alias = "s")] + Status(FyreExistingRunArgs), + /// Destroy only the VMs owned by a benchmark run. + #[command(visible_alias = "d")] + Destroy(FyreExistingRunArgs), +} + +/// Common FYRE benchmark options. +#[derive(Debug, Clone, PartialEq, Eq, Args)] +pub(crate) struct FyreRunArgs { + /// Configuration file; defaults to the eight-run built-in/external dataplane comparison. + #[arg(short = 'f', long, value_name = "FILE")] + pub(crate) file: Option, + + /// Run identifier; generated when omitted. + #[arg(short = 'i', long, value_name = "RUN_ID")] + pub(crate) run_id: Option, +} + +/// Options for an existing FYRE benchmark run. +#[derive(Debug, Clone, PartialEq, Eq, Args)] +pub(crate) struct FyreExistingRunArgs { + /// Existing run identifier. + #[arg(short = 'i', long, value_name = "RUN_ID", required = true)] + pub(crate) run_id: String, } /// Load-test options. @@ -341,6 +400,18 @@ pub(crate) struct LoadRunArgs { /// Locust duration using positive h, m, and s groups, such as 1h30m. #[arg(short = 't', long, value_parser = parse_run_time)] pub(crate) run_time: Option, + + /// Local Locust worker processes; must be greater than zero. + #[arg(short = 'w', long, value_parser = parse_positive_usize)] + pub(crate) workers: Option, + + /// Built-in gateway container memory limit, such as 16G. + #[arg(short = 'm', long, value_parser = parse_memory_limit)] + pub(crate) builtin_memory_limit: Option, + + /// Split Docker CPUs evenly between the target and Locust. + #[arg(short = 'i', long)] + pub(crate) isolate_cpus: bool, } /// Upstream live-test options. diff --git a/src/cli_public_tests.rs b/src/cli_public_tests.rs index 7c7d7f7..1f1f60b 100644 --- a/src/cli_public_tests.rs +++ b/src/cli_public_tests.rs @@ -70,7 +70,7 @@ fn command_tree_contains_only_distinct_public_workflows() { subcommands(&["stack"]), ["up", "down", "status", "logs", "config"] ); - assert_eq!(subcommands(&["load"]), ["run"]); + assert_eq!(subcommands(&["load"]), ["run", "fyre"]); assert_eq!(subcommands(&["conformance"]), ["run", "report"]); assert_eq!(subcommands(&["debug"]), ["inspect", "token"]); } @@ -88,6 +88,10 @@ fn every_public_command_renders_help() { &["probe"], &["load"], &["load", "run"], + &["load", "fyre"], + &["load", "fyre", "run"], + &["load", "fyre", "status"], + &["load", "fyre", "destroy"], &["live"], &["conformance"], &["conformance", "run"], @@ -218,6 +222,9 @@ fn load_keeps_validated_locust_settings() { users, spawn_rate, run_time, + workers, + builtin_memory_limit, + isolate_cpus, .. }), }) = parse(&[ @@ -230,6 +237,11 @@ fn load_keeps_validated_locust_settings() { "0.5", "--run-time", "1m30s", + "--workers", + "4", + "--builtin-memory-limit", + "16G", + "--isolate-cpus", ]) .command else { @@ -241,8 +253,19 @@ fn load_keeps_validated_locust_settings() { assert_eq!(users, Some(2)); assert_eq!(spawn_rate, Some(0.5)); assert_eq!(run_time.as_deref(), Some("1m30s")); + assert_eq!(workers, Some(4)); + assert_eq!(builtin_memory_limit.as_deref(), Some("16G")); + assert!(isolate_cpus); rejected(&["cf-integration", "load", "run", "--users", "0"]); + rejected(&["cf-integration", "load", "run", "--workers", "0"]); + rejected(&[ + "cf-integration", + "load", + "run", + "--builtin-memory-limit", + "0G", + ]); rejected(&["cf-integration", "load", "run", "--run-time", "1ms"]); rejected(&["cf-integration", "load", "run", "--run-time", "zero"]); rejected(&["cf-integration", "load", "run", "--engine", "locust"]); @@ -263,7 +286,9 @@ fn load_accepts_standalone_external_dataplane_mode() { panic!("expected load") }; - let LoadCommand::Run(args) = args.command; + let LoadCommand::Run(args) = args.command else { + panic!("expected load run") + }; assert_eq!(args.lane, Some(CliRoutedLane::External)); } @@ -274,7 +299,9 @@ fn load_accepts_explicit_observability() { panic!("expected load") }; - let LoadCommand::Run(args) = args.command; + let LoadCommand::Run(args) = args.command else { + panic!("expected load run") + }; assert!(args.observability); } @@ -676,7 +703,9 @@ fn load_uses_client_eras_and_rejects_version_or_server_selectors() { else { panic!("expected load") }; - let LoadCommand::Run(args) = args.command; + let LoadCommand::Run(args) = args.command else { + panic!("expected load run") + }; assert_eq!(args.client_era, expected); } for arguments in [ @@ -785,7 +814,7 @@ fn short_commands_and_options_resolve_identically_to_long_forms() { ( &[ "l", "r", "-s", "-l", "external", "-c", "modern", "-o", "-S", "-u", "20", "-r", - "5", "-t", "2m", + "5", "-t", "2m", "-w", "4", "-i", ], &[ "load", @@ -803,8 +832,42 @@ fn short_commands_and_options_resolve_identically_to_long_forms() { "5", "--run-time", "2m", + "--workers", + "4", + "--isolate-cpus", + ], + ), + ( + &["l", "r", "-l", "builtin", "-m", "16G"], + &[ + "load", + "run", + "--lane", + "builtin", + "--builtin-memory-limit", + "16G", ], ), + ( + &["l", "f", "r", "-f", "scenario.yaml", "-i", "scale-run"], + &[ + "load", + "fyre", + "run", + "--file", + "scenario.yaml", + "--run-id", + "scale-run", + ], + ), + ( + &["l", "f", "s", "-i", "scale-run"], + &["load", "fyre", "status", "--run-id", "scale-run"], + ), + ( + &["l", "f", "d", "-i", "scale-run"], + &["load", "fyre", "destroy", "--run-id", "scale-run"], + ), ( &["v", "-l", "builtin", "-p", "legacy", "-g", "protocol"], &[ diff --git a/src/helpers/config.rs b/src/helpers/config.rs index 416a2a8..5b3eb1d 100644 --- a/src/helpers/config.rs +++ b/src/helpers/config.rs @@ -3,9 +3,9 @@ use std::collections::{BTreeMap, BTreeSet}; use std::time::Duration; -use anyhow::{Context, Result, ensure}; +use anyhow::{Context, Result, bail, ensure}; use reqwest::header::{ACCEPT, CONTENT_TYPE}; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use serde_json::{Map, Value, json}; use url::Url; @@ -14,6 +14,24 @@ use crate::mcp::protocol::{self, is_stateless_protocol, jsonrpc_with_id, with_re const TIMEOUT: Duration = Duration::from_secs(10); +#[derive(Serialize)] +struct GlobalConfigKey; + +#[derive(Debug, Serialize)] +struct Authority { + hostname: String, + port: u16, +} + +#[derive(Serialize)] +struct GlobalConfig { + mcp_standard_header_max_count: Option, + mcp_standard_header_max_value_bytes: Option, + mcp_standard_header_max_total_bytes: Option, + mcp_allowed_origins: Option>, + mcp_allowed_hosts: Option>, +} + #[derive(Deserialize)] struct Tool { name: String, @@ -103,6 +121,96 @@ pub(super) async fn publish(redis_url: &str, subject: &str, body: &Value) -> Res .context("failed to publish dataplane config") } +pub(super) async fn publish_global( + redis_url: &str, + allowed_hosts: &str, + allowed_origins: &str, +) -> Result<()> { + let (key, body) = encode_global_config(allowed_hosts, allowed_origins)?; + let client = redis::Client::open(redis_url).context("invalid config Redis URL")?; + let options = redis::AsyncConnectionConfig::new() + .set_connection_timeout(Some(TIMEOUT)) + .set_response_timeout(Some(TIMEOUT)); + let mut connection = client + .get_multiplexed_async_connection_with_config(&options) + .await + .context("failed to connect to config Redis")?; + redis::cmd("SET") + .arg(key) + .arg(body) + .query_async::<()>(&mut connection) + .await + .context("failed to publish dataplane global config") +} + +pub(super) fn encode_global_config( + allowed_hosts: &str, + allowed_origins: &str, +) -> Result<(Vec, Vec)> { + let allowed_hosts = comma_separated(allowed_hosts) + .map(parse_authority) + .collect::>>()?; + let allowed_origins = comma_separated(allowed_origins) + .map(|origin| { + let url = Url::parse(origin).context("invalid MCP allowed origin")?; + ensure!( + matches!(url.scheme(), "http" | "https") + && url.host().is_some() + && url.path() == "/" + && url.query().is_none() + && url.fragment().is_none() + && url.username().is_empty() + && url.password().is_none(), + "MCP allowed origin must contain only an HTTP(S) scheme and authority" + ); + Ok(url.to_string()) + }) + .collect::>>()?; + ensure!( + !allowed_hosts.is_empty() && !allowed_origins.is_empty(), + "MCP allowed hosts and origins must not be empty" + ); + let body = GlobalConfig { + mcp_standard_header_max_count: None, + mcp_standard_header_max_value_bytes: None, + mcp_standard_header_max_total_bytes: None, + mcp_allowed_origins: Some(allowed_origins), + mcp_allowed_hosts: Some(allowed_hosts), + }; + Ok(( + rmp_serde::to_vec(&GlobalConfigKey)?, + rmp_serde::to_vec_named(&body)?, + )) +} + +fn comma_separated(value: &str) -> impl Iterator { + value + .split(',') + .map(str::trim) + .filter(|value| !value.is_empty()) +} + +fn parse_authority(value: &str) -> Result { + let url = Url::parse(&format!("http://{value}")) + .with_context(|| format!("invalid MCP allowed host {value:?}"))?; + if url.path() != "/" + || url.query().is_some() + || url.fragment().is_some() + || !url.username().is_empty() + || url.password().is_some() + { + bail!("MCP allowed host must contain only a hostname and optional port"); + } + let hostname = url + .host_str() + .context("MCP allowed host has no hostname")? + .to_owned(); + let port = url + .port_or_known_default() + .context("MCP allowed host has no port")?; + Ok(Authority { hostname, port }) +} + pub(super) async fn fixture_catalog(url: Url, version: &str) -> Result { let mut client = FixtureClient { http: reqwest::Client::builder().timeout(TIMEOUT).build()?, diff --git a/src/helpers/mod.rs b/src/helpers/mod.rs index 0b27be7..02bd9a0 100644 --- a/src/helpers/mod.rs +++ b/src/helpers/mod.rs @@ -38,6 +38,12 @@ enum HelperCommand { tenant_id: String, user_id: String, }, + GlobalConfig { + #[arg(long)] + allowed_hosts: String, + #[arg(long)] + allowed_origins: String, + }, Fixture(ConfigArgs), } @@ -92,6 +98,15 @@ pub(crate) async fn run(arguments: &[OsString]) -> Result<()> { ); return Ok(()); } + HelperCommand::GlobalConfig { + allowed_hosts, + allowed_origins, + } => { + let redis_url = std::env::var("CF_CONFIG_REDIS_URL") + .unwrap_or_else(|_| "redis://redis:6379".to_owned()); + config::publish_global(&redis_url, &allowed_hosts, &allowed_origins).await?; + return Ok(()); + } HelperCommand::Fixture(args) => args, }; ensure!( diff --git a/src/helpers/tests.rs b/src/helpers/tests.rs index 7042b6e..ffe9bf4 100644 --- a/src/helpers/tests.rs +++ b/src/helpers/tests.rs @@ -232,6 +232,39 @@ fn client_config_uses_named_messagepack_maps_and_compact_user_key() { ); } +#[test] +fn global_config_uses_the_dataplane_messagepack_contract() { + let (key, body) = config::encode_global_config( + "127.0.0.1:8080,localhost:8080,nginx", + "http://127.0.0.1:8080,http://localhost:8080", + ) + .expect("encode global config"); + assert_eq!(key, b"\x90"); + let decoded: Value = rmp_serde::from_slice(&body).expect("decode global config"); + assert_eq!( + decoded["mcp_allowed_hosts"], + json!([ + {"hostname": "127.0.0.1", "port": 8080}, + {"hostname": "localhost", "port": 8080}, + {"hostname": "nginx", "port": 80}, + ]) + ); + assert_eq!( + decoded["mcp_allowed_origins"], + json!(["http://127.0.0.1:8080/", "http://localhost:8080/"]) + ); + for field in [ + "mcp_standard_header_max_count", + "mcp_standard_header_max_value_bytes", + "mcp_standard_header_max_total_bytes", + ] { + assert!(decoded[field].is_null()); + } + assert!(config::encode_global_config("", "http://localhost:8080").is_err()); + assert!(config::encode_global_config("localhost:8080/path", "http://localhost:8080").is_err()); + assert!(config::encode_global_config("localhost:8080", "https://example.com/path").is_err()); +} + #[tokio::test] async fn auth_reuses_private_key_and_serves_only_public_jwks() { let directory = tempfile::tempdir().expect("key directory"); diff --git a/src/infrastructure/assets.rs b/src/infrastructure/assets.rs index f9ce327..843f4fa 100644 --- a/src/infrastructure/assets.rs +++ b/src/infrastructure/assets.rs @@ -31,6 +31,23 @@ static ASSETS: LazyLock> = LazyLock::new(|| { let mut assets = vec![ asset!("Cargo.toml"), asset!("Cargo.lock"), + asset!("benchmarks/fyre/scaling.yaml"), + asset!("benchmarks/fyre/vertical-low-memory.yaml"), + asset!("benchmarks/fyre/campaign.py"), + asset!("benchmarks/fyre/report.py"), + asset!("benchmarks/fyre/README.md"), + asset!("benchmarks/fyre/deploy/dataplane.compose.yaml"), + asset!("benchmarks/fyre/deploy/builtin.compose.yaml"), + asset!("benchmarks/fyre/deploy/fast-time.compose.yaml"), + asset!("benchmarks/fyre/deploy/monitor.py"), + asset!("benchmarks/fyre/deploy/register_builtin.py"), + asset!("benchmarks/fyre/deploy/run_locust.py"), + asset!("benchmarks/fyre/deploy/smoke.py"), + asset!("benchmarks/fyre/terraform/main.tf"), + asset!("benchmarks/fyre/terraform/.terraform.lock.hcl"), + asset!("benchmarks/fyre/terraform/outputs.tf"), + asset!("benchmarks/fyre/terraform/variables.tf"), + asset!("benchmarks/fyre/terraform/versions.tf"), asset!("docker/clickstack/collector.yaml"), asset!("docker/docker-compose.cf-conformance-fixture.yaml"), asset!("docker/docker-compose.cf-conformance-controlplane.yaml"), diff --git a/src/infrastructure/compose.rs b/src/infrastructure/compose.rs index 4294731..94bd955 100644 --- a/src/infrastructure/compose.rs +++ b/src/infrastructure/compose.rs @@ -25,6 +25,7 @@ pub(crate) const SERVICE_DISPLAY_NAMES: &[(&str, &str)] = &[ ("pgbouncer", "cf-pgbouncer"), ("redis", "cf-redis"), ("dataplane", "cf-dataplane"), + ("global_config_writer", "cf-dataplane-global-config-writer"), ("config_writer", "cf-dataplane-config-writer"), ("locust", "cf-locust"), ("locust_worker", "cf-locust-worker"), diff --git a/src/infrastructure/compose_integration_tests.rs b/src/infrastructure/compose_integration_tests.rs index dc4a7e7..9f4882a 100644 --- a/src/infrastructure/compose_integration_tests.rs +++ b/src/infrastructure/compose_integration_tests.rs @@ -273,9 +273,6 @@ fn dataplane_overlays_track_the_current_image_build_and_environment_contract() { "CONTEXTFORGE_DATA_PLANE_REDIS_CONNECTION_MODE", "CONTEXTFORGE_DATA_PLANE_JWKS_URL", "CONTEXTFORGE_DATA_PLANE_UPSTREAM_CONNECTION_MODE", - "CONTEXTFORGE_DATA_PLANE_USER_CONFIG_CACHE_EXPIRY_SECONDS", - "CONTEXTFORGE_GATEWAY_RS_MCP_ALLOWED_HOSTS", - "CONTEXTFORGE_GATEWAY_RS_MCP_ALLOWED_ORIGINS", ] { assert!( environment.contains_key(yaml_serde::Value::String(key.to_owned())), @@ -288,24 +285,38 @@ fn dataplane_overlays_track_the_current_image_build_and_environment_contract() { ] { assert!(!environment.contains_key(yaml_serde::Value::String(key.to_owned()))); } - assert!( - environment - [yaml_serde::Value::String("CONTEXTFORGE_GATEWAY_RS_MCP_ALLOWED_HOSTS".to_owned())] - .as_str() - .expect("MCP Host allowlist must be text") - .contains(",nginx}"), - "the default MCP Host allowlist must accept containerized Locust through nginx" + assert_eq!( + compose["services"]["dataplane"]["depends_on"]["global_config_writer"]["condition"] + .as_str(), + Some("service_completed_successfully") + ); + assert_eq!( + compose["services"]["config_writer"]["volumes"][0].as_str(), + Some("integration_auth:/keys:ro") ); assert_eq!( compose["services"]["dataplane"]["pull_policy"].as_str(), Some("${CF_DATAPLANE_PULL_POLICY:-always}") ); + assert_eq!( + compose["services"]["dataplane"]["cpuset"].as_str(), + Some("${CF_LOAD_TARGET_CPUSET:-}") + ); + for key in ["soft", "hard"] { + assert_eq!( + compose["services"]["dataplane"]["ulimits"]["nofile"][key].as_u64(), + Some(65536) + ); + } for obsolete in [ "CONTEXTFORGE_GATEWAY_RS_ADDRESS", "CONTEXTFORGE_GATEWAY_RS_REDIS_HOSTNAME", "CONTEXTFORGE_GATEWAY_RS_TOKEN_SECRET", "CONTEXTFORGE_GATEWAY_RS_UPSTREAM_CONNECTION_MODE", "CONTEXTFORGE_GATEWAY_RS_USER_CONFIG_CACHE_EXPIRY_SECONDS", + "CONTEXTFORGE_DATA_PLANE_USER_CONFIG_CACHE_EXPIRY_SECONDS", + "CONTEXTFORGE_GATEWAY_RS_MCP_ALLOWED_HOSTS", + "CONTEXTFORGE_GATEWAY_RS_MCP_ALLOWED_ORIGINS", ] { assert!( !environment.contains_key(yaml_serde::Value::String(obsolete.to_owned())), @@ -327,6 +338,30 @@ fn dataplane_overlays_track_the_current_image_build_and_environment_contract() { assert!(load_proxy.contains("worker_rlimit_nofile 65535;")); } +#[test] +fn load_generator_overlays_raise_the_open_file_limit() { + for file in [ + "docker/docker-compose.cf-integration.yaml", + "docker/docker-compose.cf-dataplane-standalone.yaml", + ] { + let compose = fs::read_to_string(workspace_root().join(file)) + .expect("read load-generator Compose overlay"); + let compose: yaml_serde::Value = + yaml_serde::from_str(&compose).expect("parse load-generator Compose overlay"); + assert_eq!( + compose["services"]["locust"]["cpuset"].as_str(), + Some("${CF_LOAD_LOCUST_CPUSET:-}") + ); + for key in ["soft", "hard"] { + assert_eq!( + compose["services"]["locust"]["ulimits"]["nofile"][key].as_u64(), + Some(65536), + "{file} must raise Locust's {key} open-file limit" + ); + } + } +} + #[test] fn controlplane_image_consumers_share_the_explicit_pull_policy() { let root = workspace_root(); @@ -414,6 +449,40 @@ fn both_external_projects_provide_the_client_conformance_config_writer() { assert_eq!(helpers[0]["profiles"][0].as_str(), Some("helpers")); assert_eq!(helpers[0]["networks"][0].as_str(), Some("mcpnet")); assert_eq!(helpers[0]["entrypoint"][1].as_str(), Some("__helper")); + + let global_helpers: Vec<_> = project + .files() + .iter() + .filter_map(|file| { + let source = fs::read_to_string(file).ok()?; + let compose: yaml_serde::Value = + yaml_serde::from_str(&source).expect("Compose YAML"); + let service = &compose["services"]["global_config_writer"]; + (!service["command"].is_null()).then(|| service.clone()) + }) + .collect(); + assert_eq!( + global_helpers.len(), + 1, + "each external project needs exactly one global config initializer" + ); + let command = global_helpers[0]["command"] + .as_sequence() + .expect("global config command"); + assert!( + command + .iter() + .any(|value| value.as_str() == Some("global-config")) + ); + assert!(command.iter().any(|value| { + value + .as_str() + .is_some_and(|value| value.contains("nginx:80")) + })); + assert_eq!( + global_helpers[0]["depends_on"]["redis"]["condition"].as_str(), + Some("service_healthy") + ); } } @@ -458,6 +527,10 @@ fn standalone_harness_owns_auth_without_dataplane_tools() { .as_str(), Some("http://127.0.0.1:4446/.well-known/jwks.json") ); + assert_eq!( + compose["services"]["dataplane"]["cpuset"].as_str(), + Some("${CF_LOAD_TARGET_CPUSET:-}") + ); assert!(compose["services"]["dataplane"]["command"].is_null()); assert!(compose["services"]["dataplane"]["volumes"].is_null()); assert_eq!( @@ -468,6 +541,11 @@ fn standalone_harness_owns_auth_without_dataplane_tools() { compose["services"]["config_writer"]["volumes"][0].as_str(), Some("standalone_auth:/keys:ro") ); + assert_eq!( + compose["services"]["dataplane"]["depends_on"]["global_config_writer"]["condition"] + .as_str(), + Some("service_completed_successfully") + ); assert!( compose["services"]["dataplane"]["environment"]["CONTEXTFORGE_DATA_PLANE_TOKEN_SECRET"] .is_null() @@ -489,7 +567,48 @@ fn standalone_harness_owns_auth_without_dataplane_tools() { compose["services"]["locust"]["image"].as_str(), Some("locustio/locust:2.46.2") ); + for service in ["dataplane", "locust"] { + for key in ["soft", "hard"] { + assert_eq!( + compose["services"][service]["ulimits"]["nofile"][key].as_u64(), + Some(65536) + ); + } + } assert!(compose["services"]["locust"]["environment"]["JWT_SECRET_KEY"].is_null()); + + for file in [ + "docker/nginx.cf-dataplane.conf", + "docker/nginx.cf-dataplane-standalone.conf.template", + ] { + let nginx = fs::read_to_string(root.join(file)).expect("read dataplane nginx config"); + assert!(nginx.contains("\"nginx\" \"nginx:80\";")); + assert!(nginx.contains("proxy_set_header Host $dataplane_host;")); + } +} + +#[test] +fn builtin_load_overlay_accepts_isolated_cpus_and_raises_locust_nofile() { + let compose = + fs::read_to_string(workspace_root().join("docker/docker-compose.cf-load-builtin.yaml")) + .expect("read built-in load overlay"); + let compose: yaml_serde::Value = + yaml_serde::from_str(&compose).expect("parse built-in load overlay"); + + assert_eq!( + compose["services"]["gateway"]["cpuset"].as_str(), + Some("${CF_LOAD_TARGET_CPUSET:-}") + ); + assert_eq!( + compose["services"]["locust"]["cpuset"].as_str(), + Some("${CF_LOAD_LOCUST_CPUSET:-}") + ); + for key in ["soft", "hard"] { + assert_eq!( + compose["services"]["locust"]["ulimits"]["nofile"][key].as_u64(), + Some(65536) + ); + } } #[test] diff --git a/src/performance/locust.rs b/src/performance/locust.rs index 465943d..ea0b4b8 100644 --- a/src/performance/locust.rs +++ b/src/performance/locust.rs @@ -21,6 +21,7 @@ const REQUEST_TIMEOUT_DEFAULT_SECONDS: &str = "60"; const REQUEST_TIMEOUT_ENV: &str = "LOCUST_REQUEST_TIMEOUT_SECONDS"; const REQUEST_TIMEOUT_ERROR: &str = "LOCUST_REQUEST_TIMEOUT_SECONDS must be a finite number greater than zero"; +const FAILED_PROCESS_MARKER: &[u8] = b"Shutting down (exit code 1)"; /// Prepared Docker Compose Locust invocation and its host report directory. #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) struct LocustCommand { @@ -69,7 +70,7 @@ impl LocustCommand { let volume = volume_argument(&report_dir); let adapter_volume = adapter_volume_argument(config.asset_root()); - let arguments = vec![ + let mut arguments = vec![ OsString::from("run"), OsString::from("--rm"), OsString::from("--no-deps"), @@ -101,11 +102,19 @@ impl LocustCommand { OsString::from(format!("--spawn-rate={}", settings.spawn_rate())), OsString::from(format!("--run-time={}", settings.run_time())), OsString::from("--headless"), + ]; + if settings.workers().get() > 1 { + arguments.push(OsString::from(format!( + "--processes={}", + settings.workers() + ))); + } + arguments.extend([ OsString::from("--html=/mnt/reports/locust_report.html"), OsString::from("--csv=/mnt/reports/locust"), OsString::from("--json-file=/mnt/reports/locust"), OsString::from("--only-summary"), - ]; + ]); let command = project.command(arguments); let mut command = command @@ -162,12 +171,19 @@ pub(crate) fn audit_reports(report_dir: &Path, bearer_token: &str) -> Result<()> let mut first_inspection_error = None; collect_report_files(report_dir, &mut files, &mut first_inspection_error); let mut tainted = Vec::new(); + let mut failed_process = false; for path in files { match fs::read(&path) { - Ok(contents) if contains_bytes(&contents, bearer_token.as_bytes()) => { - tainted.push(path); + Ok(contents) => { + if path.file_name() == Some(OsStr::new("locust.log")) + && contains_bytes(&contents, FAILED_PROCESS_MARKER) + { + failed_process = true; + } + if contains_bytes(&contents, bearer_token.as_bytes()) { + tainted.push(path); + } } - Ok(_) => {} Err(error) if first_inspection_error.is_none() => { first_inspection_error = Some((path, error)); } @@ -196,6 +212,9 @@ pub(crate) fn audit_reports(report_dir: &Path, bearer_token: &str) -> Result<()> if let Some((path, error)) = first_inspection_error { return Err(error).with_context(|| format!("failed to inspect Locust report {path:?}")); } + if failed_process { + bail!("a Locust worker exited with a failure"); + } Ok(()) } @@ -312,4 +331,19 @@ mod tests { assert!(!second.exists()); assert!(safe.is_file()); } + + #[test] + fn report_audit_rejects_a_hidden_worker_failure() { + let directory = tempfile::tempdir().expect("temporary report directory"); + fs::write( + directory.path().join("locust.log"), + "worker: Shutting down (exit code 1)\nmaster: Shutting down (exit code 0)\n", + ) + .expect("Locust log should be written"); + + let error = audit_reports(directory.path(), "absent-token") + .expect_err("a failed worker must fail the report audit"); + + assert_eq!(error.to_string(), "a Locust worker exited with a failure"); + } } diff --git a/src/performance/locust_integration_tests.rs b/src/performance/locust_integration_tests.rs index ffa27ec..3af0adc 100644 --- a/src/performance/locust_integration_tests.rs +++ b/src/performance/locust_integration_tests.rs @@ -48,6 +48,7 @@ fn args(smoke: bool) -> LoadRequest { users: None, spawn_rate: None, run_time: None, + workers: None, } } @@ -77,6 +78,7 @@ fn full_load_uses_configured_defaults() { assert_eq!(settings.users().get(), 100); assert_eq!(settings.spawn_rate(), 10.0); assert_eq!(settings.run_time(), "5m"); + assert_eq!(settings.workers().get(), 1); } #[test] @@ -91,6 +93,7 @@ fn command_line_values_override_process_environment() { cli.users = Some(3); cli.spawn_rate = Some(0.5); cli.run_time = Some(String::from("15s")); + cli.workers = Some(4); let settings = LoadSettings::resolve(&config(root.path(), &process), &cli) .expect("CLI settings should resolve"); @@ -98,6 +101,22 @@ fn command_line_values_override_process_environment() { assert_eq!(settings.users().get(), 3); assert_eq!(settings.spawn_rate(), 0.5); assert_eq!(settings.run_time(), "15s"); + assert_eq!(settings.workers().get(), 4); +} + +#[test] +fn load_worker_count_must_be_positive() { + let root = repository_root(None); + let mut request = args(false); + request.workers = Some(0); + + let error = LoadSettings::resolve(&config(root.path(), &Environment::new()), &request) + .expect_err("zero load workers should fail"); + + assert_eq!( + error.to_string(), + "load workers must be an integer greater than zero" + ); } #[test] @@ -241,6 +260,38 @@ fn dataplane_locust_command_has_exact_compose_shape_and_environment() { .contains_key(OsStr::new("COMPOSE_PROGRESS")), "Locust Compose runs must retain terminal-aware progress" ); + assert!( + !arguments + .iter() + .any(|argument| argument.to_string_lossy().starts_with("--processes=")), + "the default must preserve Locust's single-process behavior" + ); +} + +#[test] +fn multiple_load_workers_enable_locust_processes() { + let root = repository_root(None); + let config = config(root.path(), &Environment::new()); + let mut request = args(false); + request.workers = Some(4); + let settings = LoadSettings::resolve(&config, &request).expect("settings should resolve"); + + let run = LocustCommand::new( + &config, + project(&config, StackMode::Dataplane), + StackMode::Dataplane, + &settings, + "scoped.jwt.value", + Some("server-id"), + ProtocolVersion::Modern, + ) + .expect("distributed Locust command should build"); + + assert!( + run.command() + .arguments() + .contains(&OsString::from("--processes=4")) + ); } #[test] diff --git a/src/performance/python_adapter_tests.rs b/src/performance/python_adapter_tests.rs index 19a7888..984eaf1 100644 --- a/src/performance/python_adapter_tests.rs +++ b/src/performance/python_adapter_tests.rs @@ -22,8 +22,10 @@ fn workspace_root() -> PathBuf { fn locust_stub() -> TempDir { let directory = tempfile::tempdir().expect("temporary Python stub should be created"); + let locust = directory.path().join("locust"); + fs::create_dir(&locust).expect("Locust stub package should be created"); fs::write( - directory.path().join("locust.py"), + locust.join("__init__.py"), r#" class HttpUser: pass @@ -38,7 +40,7 @@ class Events: events = Events() -def between(*_args): +def constant(*_args): return lambda: None def task(_weight): @@ -46,9 +48,14 @@ def task(_weight): "#, ) .expect("Locust stub should be written"); + fs::write( + locust.join("runners.py"), + "class MasterRunner:\n pass\n\nclass WorkerRunner:\n pass\n", + ) + .expect("Locust runner stubs should be written"); fs::write( directory.path().join("gevent.py"), - "def spawn_later(_delay, callback):\n callback()\n", + "def spawn_later(_delay, callback, *args):\n callback(*args)\n", ) .expect("gevent stub"); directory @@ -61,6 +68,8 @@ fn locust_adapter_imports_and_handles_mcp_bodies() { .expect("Python path should join"); let code = r#" import json +import os +import tempfile import locustfile_mcp as adapter assert adapter.PROTOCOL_VERSION == "2026-07-28" @@ -113,12 +122,15 @@ class Total: class Stats: total = Total() + def __init__(self): self.reset_calls = 0 + def reset_all(self): self.reset_calls += 1 class Environment: stats = Stats() process_exit_code = 0 empty_environment = Environment() +empty_environment.runner = object() adapter.fail_empty_run(empty_environment) assert empty_environment.process_exit_code == 1 @@ -141,6 +153,63 @@ assert running.process_exit_code == 1 assert running.runner.stopped == 1 running.events.user_error.callback(exception=RuntimeError("user failed")) assert running.runner.stopped == 1 + +from locust.runners import MasterRunner, WorkerRunner + +class DistributedWorker(WorkerRunner): + def __init__(self): + self.messages = [] + self.stopped = 0 + self.client_id = "worker-1" + def send_message(self, kind, payload): self.messages.append((kind, payload)) + def quit(self): self.stopped += 1 + +worker = Environment() +worker.events = Events() +worker.runner = DistributedWorker() +adapter.install_fail_fast(worker) +worker.events.request.callback(exception=RuntimeError("worker request failed")) +assert worker.process_exit_code == 1 +assert worker.runner.messages == [( + adapter._FAIL_FAST_MESSAGE, + {"error": "worker request failed", "worker": "worker-1"}, +)] +assert worker.runner.stopped == 0 + +class DistributedMaster(MasterRunner): + def __init__(self): + self.listeners = {} + self.stats = Stats() + self.stopped = 0 + def register_message(self, kind, listener): self.listeners[kind] = listener + def quit(self): self.stopped += 1 + +master = Environment() +master.events = Events() +master.runner = DistributedMaster() +adapter.install_fail_fast(master) +master.runner.listeners[adapter._FAIL_FAST_MESSAGE](environment=master, msg=object()) +assert master.process_exit_code == 1 +assert master.runner.stopped == 1 + +measurement = Environment() +measurement.events = Events() +measurement.events.spawning_complete = Hook() +measurement.runner = DistributedMaster() +with tempfile.TemporaryDirectory() as directory: + marker = os.path.join(directory, "measurement-start.txt") + os.environ["MCP_MEASUREMENT_MARKER"] = marker + os.environ["MCP_MEASUREMENT_SECONDS"] = "120" + os.environ["MCP_WARMUP_SECONDS"] = "0" + adapter.install_fail_fast(measurement) + measurement.events.spawning_complete.callback(user_count=125) + assert os.path.isfile(marker) + assert float(open(marker, encoding="utf-8").read()) > 0 + assert measurement.runner.stats.reset_calls == 1 + assert measurement.runner.stopped == 1 +os.environ.pop("MCP_MEASUREMENT_MARKER") +os.environ.pop("MCP_MEASUREMENT_SECONDS") +os.environ.pop("MCP_WARMUP_SECONDS") "#; let output = Command::new(python()) diff --git a/src/performance/settings.rs b/src/performance/settings.rs index a21965f..9738dbe 100644 --- a/src/performance/settings.rs +++ b/src/performance/settings.rs @@ -22,6 +22,8 @@ pub(crate) struct LoadRequest { pub(crate) spawn_rate: Option, /// Explicit engine duration override. pub(crate) run_time: Option, + /// Explicit local load-generator worker count. + pub(crate) workers: Option, } /// Validated load settings after applying CLI, process, dotenv, and default precedence. @@ -30,6 +32,7 @@ pub(crate) struct LoadSettings { users: NonZeroUsize, spawn_rate: f64, run_time: String, + workers: NonZeroUsize, } impl LoadSettings { @@ -76,10 +79,14 @@ impl LoadSettings { )?; validate_locust_run_time(&run_time)?; + let workers = NonZeroUsize::new(request.workers.unwrap_or(1)) + .context("load workers must be an integer greater than zero")?; + Ok(Self { users, spawn_rate, run_time, + workers, }) } @@ -100,6 +107,12 @@ impl LoadSettings { pub(crate) fn run_time(&self) -> &str { &self.run_time } + + /// Returns the number of local Locust worker processes. + #[must_use] + pub(crate) fn workers(&self) -> NonZeroUsize { + self.workers + } } fn selected_value<'a>( diff --git a/src/runtime/conformance/mod.rs b/src/runtime/conformance/mod.rs index ace07db..42c7e1f 100644 --- a/src/runtime/conformance/mod.rs +++ b/src/runtime/conformance/mod.rs @@ -656,7 +656,9 @@ impl RuntimeContext { let run_routed = lanes.contains(&target); let stack_progress = Activity::spinner(format!("Prepare {}", topology.lane_label())); let mut topology_failure = if standalone_topology { - self.stack_up_standalone_dataplane(true, true).await.err() + self.stack_up_standalone_dataplane(true, true, None) + .await + .err() } else { self.stack_up_for_conformance(topology, true).await.err() }; @@ -1002,26 +1004,16 @@ impl RuntimeContext { }; let stack_progress = Activity::spinner(progress); let stack_result = if standalone { - self.stack_up_standalone_dataplane(!reuse_stack, true).await + self.stack_up_standalone_dataplane(!reuse_stack, true, None) + .await } else { self.stack_up_for_conformance(StackMode::Dataplane, !reuse_stack) .await }; stack_progress.finish(stack_result.is_ok()); let mut failure = stack_result.err(); - let mut token = None; let mut publisher_stopped = false; - if failure.is_none() { - match if standalone { - self.standalone_dataplane_token(true) - } else { - self.issue_conformance_token().await - } { - Ok(issued) => token = Some(issued), - Err(error) => failure = Some(error), - } - } if failure.is_none() && !standalone { let progress = Activity::spinner("Pause the control-plane publisher"); let result = self.set_control_plane_publisher(false).await; @@ -1032,26 +1024,16 @@ impl RuntimeContext { failure = result.err(); } if failure.is_none() { - match token.as_ref() { - Some(issued) => { - failure = self - .run_official_client_conformance( - spec_version, - server_era, - &issued.value, - paths, - cancellation, - standalone, - ) - .await - .err(); - } - None => { - failure = Some(AppFailure::from(anyhow!( - "client conformance token was not available after issuance" - ))); - } - } + failure = self + .run_official_client_conformance( + spec_version, + server_era, + paths, + cancellation, + standalone, + ) + .await + .err(); } if publisher_stopped { @@ -1060,9 +1042,6 @@ impl RuntimeContext { progress.finish(result.is_ok()); failure = finish_with_cleanup(failure, result).err(); } - if !standalone && let Some(token) = token.as_ref() { - failure = finish_with_cleanup(failure, self.revoke_managed_token(token).await).err(); - } let cleanup = if standalone { self.cleanup_standalone_dataplane(CleanupKind::Down) } else { @@ -1130,7 +1109,6 @@ impl RuntimeContext { &self, spec_version: &str, server_era: ConformanceServerEra, - token: &str, paths: &ConformancePaths, cancellation: tokio::sync::watch::Receiver, standalone: bool, @@ -1187,18 +1165,29 @@ impl RuntimeContext { standalone, )? .env(CLIENT_BASE_URL_ENV, "http://nginx") - .env(CLIENT_SERVER_ID_ENV, CLIENT_CONFORMANCE_SERVER_ID) - .env(CLIENT_TOKEN_ENV, token); + .env(CLIENT_SERVER_ID_ENV, CLIENT_CONFORMANCE_SERVER_ID); let progress = Activity::spinner(format!( "Run external dataplane client ({} scenarios)", expected_scenarios.len() )); let mut operational_failures = Vec::new(); + let mut tokens = Vec::new(); for scenario in DEFAULT_CLIENT_CONFORMANCE_SCENARIOS { + let token = match self.client_conformance_token(standalone) { + Ok(token) => token, + Err(error) => { + operational_failures.push(format!( + "{scenario}: failed to issue isolated client-conformance token: {error}" + )); + continue; + } + }; + let compose = compose.clone().env(CLIENT_TOKEN_ENV, &token.value); + tokens.push(token.value); let arguments = ["client", scenario, spec_version].map(OsString::from); let result = self .run_tool( - compose.clone(), + compose, &arguments, Some(&lane_paths.root), Some(&lane_paths.root.join(format!("runner-{scenario}.log"))), @@ -1216,11 +1205,11 @@ impl RuntimeContext { } match client_driver_failures(&lane_paths.official_results) { - Ok(failures) => operational_failures.extend( - failures - .into_iter() - .map(|failure| failure.replace(token, "[redacted]")), - ), + Ok(failures) => operational_failures.extend(failures.into_iter().map(|failure| { + tokens.iter().fold(failure, |failure, token| { + failure.replace(token, "[redacted]") + }) + })), Err(error) => operational_failures.push(error.to_string()), } diff --git a/src/runtime/fyre.rs b/src/runtime/fyre.rs new file mode 100644 index 0000000..a74c2e8 --- /dev/null +++ b/src/runtime/fyre.rs @@ -0,0 +1,1210 @@ +//! Repeatable FYRE infrastructure and benchmark-campaign orchestration. + +use std::collections::BTreeSet; +use std::ffi::{OsStr, OsString}; +use std::fs; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use anyhow::{Context, Result, bail, ensure}; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use uuid::Uuid; + +use super::{AppFailure, AppResult, CommandSpec, ProcessRunner, RuntimeContext}; +use crate::app::FyreAction; + +const OWNERSHIP_FILE: &str = "run.json"; +const TERRAFORM_DIRECTORY: &str = "terraform"; +const TERRAFORM_VARIABLES: &str = "scenario.tfvars.json"; +const HELPER_SATURATION_EXIT: i32 = 42; +const FYRE_STANDALONE_UBUNTU_OS_DISK_GB: u32 = 250; + +#[derive(Debug, Clone, Serialize, Deserialize)] +struct FyreConfig { + schema_version: u32, + #[serde(default = "default_benchmark_kind")] + benchmark_kind: String, + infrastructure: InfrastructureConfig, + images: ImageConfig, + workload: WorkloadConfig, + scenarios: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + active_helper: Option, + #[serde(skip_serializing_if = "Option::is_none")] + resolved_ssh_private_key: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +struct InfrastructureConfig { + os: String, + ssh_user: String, + ssh_private_key: PathBuf, + ssh_public_key: PathBuf, + expiry_hours: u32, + helper_sizes: Vec, + #[serde(default)] + initial_helpers: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +struct MachineSize { + cpu: u32, + memory_gb: u32, +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +struct InitialHelpers { + locust: MachineSize, + fast_time: MachineSize, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +struct ActiveHelper { + locust_cpu: u32, + locust_memory_gb: u32, + fast_time_cpu: u32, + fast_time_memory_gb: u32, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +struct ImageConfig { + dataplane: String, + #[serde(default)] + controlplane: Option, + fast_time: String, + helpers: String, + locust: String, + #[serde(default)] + postgres: Option, + redis: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +struct WorkloadConfig { + protocol_version: String, + #[serde(default)] + user_levels: Vec, + first_users: u32, + maximum_users: u32, + ramp_seconds: u32, + warmup_seconds: u32, + measure_seconds: u32, + repetitions: u32, + maximum_campaign_seconds: u32, + plateau_improvement_percent: f64, + boundary_percent: f64, + config_cache_seconds: u32, + helper_cpu_percent: f64, + helper_memory_percent: f64, + worker_core_percent: f64, + tools: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +struct Scenario { + id: String, + label: String, + replicas: u32, + cpu: u32, + memory_gb: u32, + multiplier: u32, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +struct FyreQuota { + product_group_id: u64, + product_group_name: String, + cpu: u32, + cpu_used: u32, + memory: u32, + memory_used: u32, + disk: u32, + disk_used: u32, + public_ips: u32, + public_ips_used: u32, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct RequiredCapacity { + cpu: u32, + memory: u32, + disk: u32, + public_ips: u32, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +struct RunState { + schema_version: u32, + run_id: String, + phase: String, + config_file: PathBuf, + current_scenario: Option, + locust_helper_size: usize, + fast_time_helper_size: usize, + completed_scenarios: Vec, + cleanup_required: bool, +} + +impl RuntimeContext { + pub(super) async fn execute_fyre(&self, action: FyreAction) -> AppResult<()> { + match action { + FyreAction::Run { file, run_id } => self.run_fyre(file, run_id).await, + FyreAction::Status { run_id } => self.fyre_status(&run_id), + FyreAction::Destroy { run_id } => self.destroy_fyre(&run_id).await, + } + } + + async fn run_fyre(&self, file: Option, run_id: Option) -> AppResult<()> { + let source = file.unwrap_or_else(|| { + self.config + .asset_root() + .join("benchmarks/fyre/scaling.yaml") + }); + let mut config = read_config(&source).map_err(AppFailure::from)?; + validate_config(&config).map_err(AppFailure::from)?; + self.require_fyre_credentials()?; + let private_key = + expand_home(&config.infrastructure.ssh_private_key).map_err(AppFailure::from)?; + let public_key = + expand_home(&config.infrastructure.ssh_public_key).map_err(AppFailure::from)?; + ensure_file(&private_key, "SSH private key").map_err(AppFailure::from)?; + ensure_file(&public_key, "SSH public key").map_err(AppFailure::from)?; + config.resolved_ssh_private_key = Some(private_key); + + let run_id = run_id + .unwrap_or_else(|| format!("scale-{}", &Uuid::new_v4().simple().to_string()[..12])); + validate_run_id(&run_id).map_err(AppFailure::from)?; + let root = self.config.integration_dir().join("fyre").join(&run_id); + if root.exists() { + return Err(AppFailure::from(anyhow::anyhow!( + "FYRE run {run_id} already exists at {}; use status or destroy with this run ID", + root.display() + ))); + } + fs::create_dir_all(root.join("results")) + .with_context(|| format!("failed to create FYRE run directory {}", root.display())) + .map_err(AppFailure::from)?; + copy_tree( + &self.config.asset_root().join("benchmarks/fyre/terraform"), + &root.join(TERRAFORM_DIRECTORY), + ) + .map_err(AppFailure::from)?; + let config_path = root.join("config.json"); + write_json(&config_path, &config).map_err(AppFailure::from)?; + let (locust_helper_size, fast_time_helper_size) = + initial_helper_indices(&config).map_err(AppFailure::from)?; + let mut state = RunState { + schema_version: 1, + run_id: run_id.clone(), + phase: "initializing".to_owned(), + config_file: source, + current_scenario: None, + locust_helper_size, + fast_time_helper_size, + completed_scenarios: Vec::new(), + cleanup_required: true, + }; + write_state(&root, &state).map_err(AppFailure::from)?; + + let terraform = terraform_binary().map_err(AppFailure::from)?; + let init = self.fyre_environment( + CommandSpec::new(&terraform) + .args(["init", "-input=false"]) + .cwd(root.join(TERRAFORM_DIRECTORY)), + ); + let primary = async { + self.run_cancellable(&init).await?; + let validate = self.fyre_environment( + CommandSpec::new(&terraform) + .arg("validate") + .cwd(root.join(TERRAFORM_DIRECTORY)), + ); + self.run_cancellable(&validate).await?; + self.preflight_fyre_quota(&terraform, &root, &public_key, &mut config, &mut state) + .await?; + let matrix = tokio::time::timeout( + Duration::from_secs(config.workload.maximum_campaign_seconds.into()), + self.run_fyre_matrix( + &terraform, + &root, + &config_path, + &public_key, + &mut config, + &mut state, + ), + ) + .await; + match matrix { + Ok(result) => result?, + Err(_) => { + if let Some(scenario) = state.current_scenario.as_deref() { + let _ = self + .collect_fyre_scenario(&root, &config_path, scenario) + .await; + } + return Err(AppFailure::from(anyhow::anyhow!( + "FYRE campaign exceeded its configured time bound" + ))); + } + } + write_json( + &root.join("manifest.json"), + &json!({ + "schema_version": 1, + "run_id": run_id, + "configuration": config, + "state": state, + "terraform_lock": root.join(TERRAFORM_DIRECTORY).join(".terraform.lock.hcl"), + }), + ) + .map_err(AppFailure::from) + } + .await; + + state.phase = "collecting".to_owned(); + let _ = write_state(&root, &state); + let report_result = if primary.is_ok() { + self.generate_fyre_report(&root, &config_path).await + } else { + Ok(()) + }; + let primary = primary.and(report_result); + state.phase = "destroying".to_owned(); + let _ = write_state(&root, &state); + let cleanup = self.terraform_destroy(&terraform, &root).await; + if cleanup.is_ok() { + state.cleanup_required = false; + state.phase = if primary.is_ok() { + "complete" + } else { + "failed" + } + .to_owned(); + } else { + state.phase = "cleanup-failed".to_owned(); + } + let _ = write_state(&root, &state); + super::finish_with_cleanup(primary.err(), cleanup) + } + + async fn preflight_fyre_quota( + &self, + terraform: &OsString, + root: &Path, + public_key: &Path, + config: &mut FyreConfig, + state: &mut RunState, + ) -> AppResult<()> { + let locust = config.infrastructure.helper_sizes[state.locust_helper_size]; + let fast_time = config.infrastructure.helper_sizes[state.fast_time_helper_size]; + config.active_helper = Some(ActiveHelper { + locust_cpu: locust.cpu, + locust_memory_gb: locust.memory_gb, + fast_time_cpu: fast_time.cpu, + fast_time_memory_gb: fast_time.memory_gb, + }); + write_json(&root.join("config.json"), config).map_err(AppFailure::from)?; + state.phase = "checking-quota".to_owned(); + write_state(root, state).map_err(AppFailure::from)?; + + let public_key = fs::read_to_string(public_key) + .context("failed to read FYRE SSH public key") + .map_err(AppFailure::from)?; + let scenario = config + .scenarios + .first() + .expect("validated FYRE configuration has a baseline scenario"); + let variables = terraform_variables( + &state.run_id, + config, + scenario, + &public_key, + self.fyre_text("FYRE_PRODUCT_GROUP_ID"), + self.fyre_text("FYRE_SITE"), + ); + write_json(&root.join(TERRAFORM_VARIABLES), &variables).map_err(AppFailure::from)?; + + let refresh = self.fyre_environment( + CommandSpec::new(terraform) + .args([ + "apply", + "-refresh-only", + "-input=false", + "-auto-approve", + "-var-file", + ]) + .arg(root.join(TERRAFORM_VARIABLES)) + .cwd(root.join(TERRAFORM_DIRECTORY)), + ); + self.run_cancellable(&refresh).await?; + let quota = self.terraform_quota(terraform, root)?; + write_json(&root.join("quota.json"), "a).map_err(AppFailure::from)?; + ensure_fyre_quota(config, "a).map_err(AppFailure::from) + } + + #[allow(clippy::too_many_arguments)] + async fn run_fyre_matrix( + &self, + terraform: &OsString, + root: &Path, + config_path: &Path, + public_key: &Path, + config: &mut FyreConfig, + state: &mut RunState, + ) -> AppResult<()> { + let public_key = fs::read_to_string(public_key) + .context("failed to read FYRE SSH public key") + .map_err(AppFailure::from)?; + let mut index = 0; + while index < config.scenarios.len() { + let scenario = config.scenarios[index].clone(); + let locust = config.infrastructure.helper_sizes[state.locust_helper_size]; + let fast_time = config.infrastructure.helper_sizes[state.fast_time_helper_size]; + config.active_helper = Some(ActiveHelper { + locust_cpu: locust.cpu, + locust_memory_gb: locust.memory_gb, + fast_time_cpu: fast_time.cpu, + fast_time_memory_gb: fast_time.memory_gb, + }); + write_json(config_path, config).map_err(AppFailure::from)?; + state.current_scenario = Some(scenario.id.clone()); + state.phase = "provisioning".to_owned(); + write_state(root, state).map_err(AppFailure::from)?; + let variables = terraform_variables( + &state.run_id, + config, + &scenario, + &public_key, + self.fyre_text("FYRE_PRODUCT_GROUP_ID"), + self.fyre_text("FYRE_SITE"), + ); + write_json(&root.join(TERRAFORM_VARIABLES), &variables).map_err(AppFailure::from)?; + let apply = self.fyre_environment( + CommandSpec::new(terraform) + .args(["apply", "-input=false", "-auto-approve", "-var-file"]) + .arg(root.join(TERRAFORM_VARIABLES)) + .cwd(root.join(TERRAFORM_DIRECTORY)), + ); + self.run_cancellable(&apply).await?; + let inventory = self.terraform_inventory(terraform, root)?; + let scenario_root = root.join("results").join(&scenario.id); + if scenario_root.exists() { + fs::remove_dir_all(&scenario_root) + .with_context(|| format!("failed to reset {}", scenario_root.display())) + .map_err(AppFailure::from)?; + } + fs::create_dir_all(&scenario_root) + .with_context(|| format!("failed to create {}", scenario_root.display())) + .map_err(AppFailure::from)?; + let inventory_path = scenario_root.join("inventory.json"); + write_json(&inventory_path, &inventory).map_err(AppFailure::from)?; + state.phase = "benchmarking".to_owned(); + write_state(root, state).map_err(AppFailure::from)?; + let campaign = self.fyre_campaign_command(root, config_path, &scenario.id); + let campaign_result = self.run_cancellable(&campaign).await; + if campaign_result.is_err() { + let _ = self + .collect_fyre_scenario(root, config_path, &scenario.id) + .await; + } + match campaign_result { + Ok(()) => { + state.completed_scenarios.push(scenario.id); + write_state(root, state).map_err(AppFailure::from)?; + index += 1; + } + Err(AppFailure::Infrastructure( + crate::infrastructure::InfrastructureError::ChildExit { status, .. }, + )) if status.code() == Some(HELPER_SATURATION_EXIT) => { + let request: Value = serde_json::from_slice( + &fs::read(scenario_root.join("helper-request.json")) + .context("helper saturation did not produce helper-request.json") + .map_err(AppFailure::from)?, + ) + .context("invalid helper saturation request") + .map_err(AppFailure::from)?; + let role = request["role"].as_str().ok_or_else(|| { + AppFailure::from(anyhow::anyhow!( + "helper saturation request has an unknown role" + )) + })?; + let size = match role { + "locust" => &mut state.locust_helper_size, + "fast-time" => &mut state.fast_time_helper_size, + _ => { + return Err(AppFailure::from(anyhow::anyhow!( + "helper saturation request has an unknown role" + ))); + } + }; + *size += 1; + if *size >= config.infrastructure.helper_sizes.len() { + return Err(AppFailure::from(anyhow::anyhow!( + "helper headroom is inconclusive: the saturated helper reached the configured 16 vCPU / 32 GB limit" + ))); + } + let archive = root + .join("invalidated") + .join(format!("{role}-size-{}-at-{}", *size, scenario.id)); + fs::create_dir_all(&archive) + .with_context(|| format!("failed to create {}", archive.display())) + .map_err(AppFailure::from)?; + for scenario in &config.scenarios { + let path = root.join("results").join(&scenario.id); + if path.exists() { + let archived = archive.join(&scenario.id); + fs::rename(&path, &archived) + .with_context(|| { + format!( + "failed to archive {} as {}", + path.display(), + archived.display() + ) + }) + .map_err(AppFailure::from)?; + } + } + state.completed_scenarios.clear(); + index = 0; + } + Err(error) => return Err(error), + } + } + state.current_scenario = None; + Ok(()) + } + + fn fyre_campaign_command( + &self, + root: &Path, + config_path: &Path, + scenario: &str, + ) -> CommandSpec { + let scenario_root = root.join("results").join(scenario); + CommandSpec::new("python3") + .arg(self.config.asset_root().join("benchmarks/fyre/campaign.py")) + .arg("--config") + .arg(config_path) + .arg("--inventory") + .arg(scenario_root.join("inventory.json")) + .args(["--scenario", scenario]) + .arg("--deploy") + .arg(self.config.asset_root().join("benchmarks/fyre/deploy")) + .arg("--ansible") + .arg( + self.config + .asset_root() + .join("benchmarks/fyre/ansible/bootstrap.yml"), + ) + .arg("--output") + .arg(scenario_root) + } + + async fn collect_fyre_scenario( + &self, + root: &Path, + config_path: &Path, + scenario: &str, + ) -> AppResult<()> { + let inventory = root.join("results").join(scenario).join("inventory.json"); + if !inventory.is_file() { + return Ok(()); + } + let collection = self + .fyre_campaign_command(root, config_path, scenario) + .arg("--collect-only"); + self.run_cancellable(&collection).await + } + + fn terraform_inventory(&self, terraform: &OsString, root: &Path) -> AppResult { + let command = self.fyre_environment( + CommandSpec::new(terraform) + .args(["output", "-json", "inventory"]) + .cwd(root.join(TERRAFORM_DIRECTORY)), + ); + let output = self + .runner + .capture_stdout(&command) + .map_err(AppFailure::from)?; + serde_json::from_slice(&output) + .context("Terraform inventory output is not valid JSON") + .map_err(AppFailure::from) + } + + fn terraform_quota(&self, terraform: &OsString, root: &Path) -> AppResult { + let command = self.fyre_environment( + CommandSpec::new(terraform) + .args(["output", "-json", "quota"]) + .cwd(root.join(TERRAFORM_DIRECTORY)), + ); + let output = self + .runner + .capture_stdout(&command) + .map_err(AppFailure::from)?; + serde_json::from_slice(&output) + .context("Terraform FYRE quota output is not valid JSON") + .map_err(AppFailure::from) + } + + async fn generate_fyre_report(&self, root: &Path, config: &Path) -> AppResult<()> { + let command = CommandSpec::new("uv") + .args(["run", "--with", "matplotlib==3.10.6"]) + .arg(self.config.asset_root().join("benchmarks/fyre/report.py")) + .arg("--config") + .arg(config) + .arg("--results") + .arg(root.join("results")); + self.run_cancellable(&command).await + } + + fn fyre_status(&self, run_id: &str) -> AppResult<()> { + validate_run_id(run_id).map_err(AppFailure::from)?; + let root = self.config.integration_dir().join("fyre").join(run_id); + let state = read_owned_state(&root, run_id).map_err(AppFailure::from)?; + println!( + "{}", + serde_json::to_string_pretty(&state) + .map_err(anyhow::Error::from) + .map_err(AppFailure::from)? + ); + Ok(()) + } + + async fn destroy_fyre(&self, run_id: &str) -> AppResult<()> { + validate_run_id(run_id).map_err(AppFailure::from)?; + let root = self.config.integration_dir().join("fyre").join(run_id); + let mut state = read_owned_state(&root, run_id).map_err(AppFailure::from)?; + let terraform = terraform_binary().map_err(AppFailure::from)?; + state.phase = "destroying".to_owned(); + write_state(&root, &state).map_err(AppFailure::from)?; + self.terraform_destroy(&terraform, &root).await?; + state.phase = "destroyed".to_owned(); + state.cleanup_required = false; + write_state(&root, &state).map_err(AppFailure::from) + } + + async fn terraform_destroy(&self, terraform: &OsString, root: &Path) -> AppResult<()> { + let variables = root.join(TERRAFORM_VARIABLES); + if !variables.is_file() { + return Ok(()); + } + let command = self.fyre_environment( + CommandSpec::new(terraform) + .args(["destroy", "-input=false", "-auto-approve", "-var-file"]) + .arg(variables) + .cwd(root.join(TERRAFORM_DIRECTORY)), + ); + let mut last = None; + for attempt in 0..3 { + match self.run_cancellable(&command).await { + Ok(()) => return Ok(()), + Err(error) => last = Some(error), + } + if attempt < 2 { + tokio::time::sleep(Duration::from_secs(2_u64.pow(attempt + 1))).await; + } + } + Err(last.unwrap_or_else(|| AppFailure::from(anyhow::anyhow!("Terraform destroy failed")))) + } + + async fn run_cancellable(&self, command: &CommandSpec) -> AppResult<()> { + let (sender, receiver) = tokio::sync::watch::channel(false); + let process = self.runner.run_async_cancellable(command, receiver); + tokio::pin!(process); + tokio::select! { + result = &mut process => result.map_err(AppFailure::from), + signal = tokio::signal::ctrl_c() => { + signal.context("failed to install interrupt handler").map_err(AppFailure::from)?; + sender.send_replace(true); + process.await.map_err(AppFailure::from) + } + } + } + + fn fyre_environment(&self, mut command: CommandSpec) -> CommandSpec { + for key in [ + "FYRE_USERNAME", + "FYRE_API_KEY", + "FYRE_PRODUCT_GROUP_ID", + "FYRE_SITE", + ] { + if let Some(value) = self + .config + .environment() + .get(OsStr::new(key)) + .map(|value| value.value.clone()) + { + command = command.env(key, value); + } + } + command + } + + fn fyre_text(&self, key: &str) -> Option<&str> { + self.config + .environment() + .get(OsStr::new(key)) + .and_then(|value| value.value.to_str()) + .filter(|value| !value.is_empty()) + } + + fn require_fyre_credentials(&self) -> AppResult<()> { + for key in ["FYRE_USERNAME", "FYRE_API_KEY"] { + if self.fyre_text(key).is_none() { + return Err(AppFailure::from(anyhow::anyhow!( + "{key} is required for FYRE provisioning" + ))); + } + } + Ok(()) + } +} + +fn terraform_variables( + run_id: &str, + config: &FyreConfig, + scenario: &Scenario, + public_key: &str, + product_group_id: Option<&str>, + site: Option<&str>, +) -> Value { + let helpers = config + .active_helper + .as_ref() + .expect("active helper must be set"); + json!({ + "run_id": run_id, + "os": config.infrastructure.os, + "ssh_public_key": public_key.trim(), + "expiry_hours": config.infrastructure.expiry_hours, + "dataplane_count": scenario.replicas, + "dataplane_cpu": scenario.cpu, + "dataplane_memory_gb": scenario.memory_gb, + "locust_cpu": helpers.locust_cpu, + "locust_memory_gb": helpers.locust_memory_gb, + "fast_time_cpu": helpers.fast_time_cpu, + "fast_time_memory_gb": helpers.fast_time_memory_gb, + "product_group_id": product_group_id, + "site": site, + }) +} + +fn read_config(path: &Path) -> Result { + let source = fs::read(path) + .with_context(|| format!("failed to read FYRE configuration {}", path.display()))?; + yaml_serde::from_slice(&source) + .with_context(|| format!("failed to parse FYRE configuration {}", path.display())) +} + +fn default_benchmark_kind() -> String { + "scaling".to_owned() +} + +fn validate_config(config: &FyreConfig) -> Result<()> { + ensure!( + config.schema_version == 1, + "unsupported FYRE configuration schema" + ); + ensure!( + config.infrastructure.os == "Ubuntu 24.04", + "FYRE benchmark OS must be Ubuntu 24.04" + ); + ensure!( + (8..=24).contains(&config.infrastructure.expiry_hours), + "FYRE expiry must be between eight and 24 hours" + ); + ensure!( + config.workload.protocol_version == "2026-07-28", + "FYRE load supports only modern 2026-07-28" + ); + ensure!( + config.workload.first_users >= 125, + "FYRE load must start at 125 users or more" + ); + ensure!( + config.workload.maximum_users <= 32_000, + "FYRE load must be bounded at 32,000 users" + ); + ensure!( + config.workload.maximum_campaign_seconds <= 36_000, + "FYRE campaign must be bounded at ten hours" + ); + match config.benchmark_kind.as_str() { + "scaling" => ensure!( + config.workload.repetitions == 3, + "candidate capacity must use three repetitions" + ), + "comparison" => { + ensure!( + config.workload.repetitions == 1, + "the fixed comparison runs each benchmark once" + ); + ensure!( + config.workload.user_levels == [125, 250, 500, 1_000], + "the default comparison must run 125, 250, 500, and 1,000 users" + ); + ensure!( + config.workload.measure_seconds == 3_600, + "each default comparison benchmark must measure for one hour" + ); + ensure!( + config.scenarios.len() == 1 && config.scenarios[0].replicas == 1, + "the comparison target must be exactly one VM" + ); + ensure!( + config.images.controlplane.is_some() && config.images.postgres.is_some(), + "the comparison requires pinned control-plane and PostgreSQL images" + ); + } + other => bail!("unsupported FYRE benchmark kind {other}"), + } + let expected_tools = BTreeSet::from([ + "convert_time", + "echo", + "get_stats", + "get_system_time", + "schema_success", + "verify-protocol", + ]); + let actual_tools = config + .workload + .tools + .iter() + .map(String::as_str) + .collect::>(); + ensure!( + actual_tools == expected_tools, + "FYRE workload must contain the six nonfailure Fast Time tools" + ); + ensure!( + !config.infrastructure.helper_sizes.is_empty(), + "at least one helper size is required" + ); + initial_helper_indices(config)?; + let maximum = config + .infrastructure + .helper_sizes + .last() + .expect("nonempty helper sizes"); + ensure!( + maximum.cpu <= 16 && maximum.memory_gb <= 32, + "helper resources exceed 16 vCPU / 32 GB" + ); + let mut ids = BTreeSet::<&str>::new(); + let baseline = config + .scenarios + .iter() + .find(|scenario| scenario.multiplier == 1) + .context("FYRE matrix requires a multiplier-one baseline")?; + ensure!( + baseline.multiplier == 1, + "baseline scenario multiplier must be one" + ); + let baseline_cpu = baseline.replicas * baseline.cpu; + let baseline_memory = baseline.replicas * baseline.memory_gb; + for scenario in &config.scenarios { + validate_run_id(&scenario.id)?; + ensure!( + ids.insert(scenario.id.as_str()), + "duplicate FYRE scenario {}", + scenario.id + ); + ensure!( + scenario.replicas > 0 && scenario.cpu > 0 && scenario.memory_gb > 0, + "scenario {} has zero resources", + scenario.id + ); + ensure!( + scenario.multiplier > 0, + "scenario {} has a zero multiplier", + scenario.id + ); + ensure!( + scenario.replicas * scenario.cpu == scenario.multiplier * baseline_cpu, + "scenario {} CPU total does not match its multiplier", + scenario.id + ); + ensure!( + scenario.replicas * scenario.memory_gb == scenario.multiplier * baseline_memory, + "scenario {} memory total does not match its multiplier", + scenario.id + ); + } + let mut images = vec![ + &config.images.dataplane, + &config.images.fast_time, + &config.images.helpers, + &config.images.locust, + &config.images.redis, + ]; + images.extend(config.images.controlplane.iter()); + images.extend(config.images.postgres.iter()); + for image in images { + ensure!( + image.contains("@sha256:"), + "all benchmark images must be pinned by digest" + ); + } + Ok(()) +} + +fn initial_helper_indices(config: &FyreConfig) -> Result<(usize, usize)> { + let Some(initial) = config.infrastructure.initial_helpers else { + return Ok((0, 0)); + }; + let find = |role: &str, size: MachineSize| { + config + .infrastructure + .helper_sizes + .iter() + .position(|candidate| *candidate == size) + .with_context(|| { + format!( + "initial {role} helper {} vCPU / {} GB is not present in helper_sizes", + size.cpu, size.memory_gb + ) + }) + }; + Ok(( + find("Locust", initial.locust)?, + find("Fast Time", initial.fast_time)?, + )) +} + +fn required_capacity(config: &FyreConfig) -> RequiredCapacity { + let maximum_replicas = config + .scenarios + .iter() + .map(|scenario| scenario.replicas) + .max() + .unwrap_or_default(); + let dataplane_cpu = config + .scenarios + .iter() + .map(|scenario| scenario.replicas * scenario.cpu) + .max() + .unwrap_or_default(); + let dataplane_memory = config + .scenarios + .iter() + .map(|scenario| scenario.replicas * scenario.memory_gb) + .max() + .unwrap_or_default(); + let maximum = config + .infrastructure + .helper_sizes + .iter() + .copied() + .max_by_key(|size| (size.cpu, size.memory_gb)) + .unwrap_or(MachineSize { + cpu: 0, + memory_gb: 0, + }); + let (locust, fast_time) = (maximum, maximum); + let vm_count = maximum_replicas + 2; + RequiredCapacity { + cpu: dataplane_cpu + locust.cpu + fast_time.cpu, + memory: dataplane_memory + locust.memory_gb + fast_time.memory_gb, + disk: vm_count * FYRE_STANDALONE_UBUNTU_OS_DISK_GB, + public_ips: vm_count, + } +} + +fn ensure_fyre_quota(config: &FyreConfig, quota: &FyreQuota) -> Result<()> { + let required = required_capacity(config); + let available = RequiredCapacity { + cpu: quota.cpu.saturating_sub(quota.cpu_used), + memory: quota.memory.saturating_sub(quota.memory_used), + disk: quota.disk.saturating_sub(quota.disk_used), + public_ips: quota.public_ips.saturating_sub(quota.public_ips_used), + }; + let mut shortages = Vec::new(); + for (name, unit, needed, free, total, used) in [ + ( + "CPU", + "vCPU", + required.cpu, + available.cpu, + quota.cpu, + quota.cpu_used, + ), + ( + "memory", + "GB", + required.memory, + available.memory, + quota.memory, + quota.memory_used, + ), + ( + "disk", + "GB", + required.disk, + available.disk, + quota.disk, + quota.disk_used, + ), + ( + "public IPs", + "addresses", + required.public_ips, + available.public_ips, + quota.public_ips, + quota.public_ips_used, + ), + ] { + if needed > free { + shortages.push(format!( + "{name} requires {needed} {unit}, but only {free} are available ({total} total, {used} used; shortage {})", + needed - free + )); + } + } + if shortages.is_empty() { + return Ok(()); + } + bail!( + "FYRE product group {} ({}) cannot fit the configured campaign: {}. The standalone VM API and pinned Terraform provider allocate a {} GB Ubuntu 24.04 root disk and expose no create-time root-disk setting; the OCP cluster API's base_disk_size setting does not apply to these standalone VMs", + quota.product_group_id, + quota.product_group_name, + shortages.join("; "), + FYRE_STANDALONE_UBUNTU_OS_DISK_GB, + ) +} + +fn validate_run_id(run_id: &str) -> Result<()> { + ensure!( + !run_id.is_empty() + && run_id.len() <= 48 + && run_id + .bytes() + .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'-') + && run_id + .as_bytes() + .first() + .is_some_and(u8::is_ascii_alphanumeric) + && run_id + .as_bytes() + .last() + .is_some_and(u8::is_ascii_alphanumeric), + "run ID must contain only lowercase letters, digits, and internal hyphens" + ); + Ok(()) +} + +fn expand_home(path: &Path) -> Result { + let text = path.to_str().context("SSH key path must be UTF-8")?; + if text == "~" || text.starts_with("~/") { + let home = std::env::var_os("HOME").context("HOME is required to expand SSH key paths")?; + return Ok(PathBuf::from(home).join(text.trim_start_matches("~/"))); + } + Ok(path.to_path_buf()) +} + +fn ensure_file(path: &Path, label: &str) -> Result<()> { + ensure!(path.is_file(), "{label} {} does not exist", path.display()); + Ok(()) +} + +fn terraform_binary() -> Result { + if let Some(binary) = std::env::var_os("CF_TERRAFORM_BIN") { + ensure!(!binary.is_empty(), "CF_TERRAFORM_BIN must not be empty"); + return Ok(binary); + } + if std::process::Command::new("terraform") + .arg("version") + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .status() + .is_ok_and(|status| status.success()) + { + return Ok(OsString::from("terraform")); + } + bail!("Terraform is required; set CF_TERRAFORM_BIN to its executable") +} + +fn copy_tree(source: &Path, destination: &Path) -> Result<()> { + fs::create_dir_all(destination) + .with_context(|| format!("failed to create {}", destination.display()))?; + for entry in + fs::read_dir(source).with_context(|| format!("failed to read {}", source.display()))? + { + let entry = entry?; + let target = destination.join(entry.file_name()); + if entry.file_type()?.is_dir() { + copy_tree(&entry.path(), &target)?; + } else { + fs::copy(entry.path(), &target) + .with_context(|| format!("failed to copy {}", entry.path().display()))?; + } + } + Ok(()) +} + +fn write_json(path: &Path, value: &impl Serialize) -> Result<()> { + let temporary = path.with_extension("tmp"); + fs::write(&temporary, serde_json::to_vec_pretty(value)?) + .with_context(|| format!("failed to write {}", temporary.display()))?; + fs::rename(&temporary, path).with_context(|| format!("failed to activate {}", path.display())) +} + +fn write_state(root: &Path, state: &RunState) -> Result<()> { + write_json(&root.join(OWNERSHIP_FILE), state) +} + +fn read_owned_state(root: &Path, expected_run_id: &str) -> Result { + let path = root.join(OWNERSHIP_FILE); + let state: RunState = serde_json::from_slice( + &fs::read(&path) + .with_context(|| format!("FYRE run state {} does not exist", path.display()))?, + ) + .context("invalid FYRE run state")?; + ensure!( + state.run_id == expected_run_id, + "FYRE run ownership mismatch; refusing cleanup" + ); + ensure!( + root.join(TERRAFORM_DIRECTORY).is_dir(), + "FYRE Terraform state directory is missing; refusing cleanup" + ); + Ok(state) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn packaged_default_is_the_eight_run_comparison() { + let config = read_config( + Path::new(env!("CARGO_MANIFEST_DIR")) + .join("benchmarks/fyre/scaling.yaml") + .as_path(), + ) + .expect("packaged FYRE config"); + validate_config(&config).expect("valid FYRE config"); + assert_eq!(config.benchmark_kind, "comparison"); + assert_eq!(config.workload.user_levels, [125, 250, 500, 1_000]); + assert_eq!(config.workload.measure_seconds, 3_600); + assert_eq!(config.workload.protocol_version, "2026-07-28"); + assert_eq!(config.scenarios.len(), 1); + } + + #[test] + fn comparison_accepts_a_configured_two_vcpu_two_gb_target() { + let mut config = read_config( + Path::new(env!("CARGO_MANIFEST_DIR")) + .join("benchmarks/fyre/scaling.yaml") + .as_path(), + ) + .expect("packaged FYRE config"); + config.scenarios[0].cpu = 2; + config.scenarios[0].memory_gb = 2; + + validate_config(&config).expect("valid 2 vCPU / 2 GB comparison target"); + } + + #[test] + fn packaged_low_memory_vertical_profile_is_valid_and_matched() { + let config = read_config( + Path::new(env!("CARGO_MANIFEST_DIR")) + .join("benchmarks/fyre/vertical-low-memory.yaml") + .as_path(), + ) + .expect("packaged low-memory FYRE config"); + validate_config(&config).expect("valid low-memory FYRE config"); + assert_eq!(config.scenarios.len(), 2); + assert_eq!( + initial_helper_indices(&config).expect("valid initial helper indices"), + (1, 2) + ); + assert_eq!( + required_capacity(&config), + RequiredCapacity { + cpu: 36, + memory: 68, + disk: 750, + public_ips: 3, + } + ); + } + + #[test] + fn cleanup_requires_matching_owned_state() { + let directory = tempfile::tempdir().expect("temporary directory"); + fs::create_dir(directory.path().join(TERRAFORM_DIRECTORY)).expect("terraform directory"); + let state = RunState { + schema_version: 1, + run_id: "owned-run".to_owned(), + phase: "failed".to_owned(), + config_file: PathBuf::from("config.yaml"), + current_scenario: None, + locust_helper_size: 0, + fast_time_helper_size: 0, + completed_scenarios: Vec::new(), + cleanup_required: true, + }; + write_state(directory.path(), &state).expect("state"); + let error = + read_owned_state(directory.path(), "another-run").expect_err("ownership mismatch"); + assert!(error.to_string().contains("ownership mismatch")); + } + + #[test] + fn run_ids_reject_paths_and_uppercase() { + for invalid in ["../manual-vm", "UPPER", "-leading", "trailing-"] { + assert!(validate_run_id(invalid).is_err(), "{invalid}"); + } + } + + #[test] + fn packaged_comparison_requires_three_standalone_vms() { + let config = read_config( + Path::new(env!("CARGO_MANIFEST_DIR")) + .join("benchmarks/fyre/scaling.yaml") + .as_path(), + ) + .expect("packaged FYRE config"); + assert_eq!( + required_capacity(&config), + RequiredCapacity { + cpu: 36, + memory: 68, + disk: 750, + public_ips: 3, + } + ); + } + + #[test] + fn quota_preflight_reports_standalone_vm_disk_shortage() { + let config = read_config( + Path::new(env!("CARGO_MANIFEST_DIR")) + .join("benchmarks/fyre/scaling.yaml") + .as_path(), + ) + .expect("packaged FYRE config"); + let error = ensure_fyre_quota( + &config, + &FyreQuota { + product_group_id: 808, + product_group_name: "benchmark".to_owned(), + cpu: 316, + cpu_used: 248, + memory: 632, + memory_used: 528, + disk: 8_000, + disk_used: 7_500, + public_ips: 50, + public_ips_used: 0, + }, + ) + .expect_err("disk quota must be rejected"); + let message = error.to_string(); + assert!(message.contains("disk requires 750 GB")); + assert!(message.contains("shortage 250")); + assert!(message.contains("allocate a 250 GB Ubuntu 24.04 root disk")); + assert!(message.contains("OCP cluster API's base_disk_size")); + } +} diff --git a/src/runtime/mod.rs b/src/runtime/mod.rs index ab47b60..5236c0b 100644 --- a/src/runtime/mod.rs +++ b/src/runtime/mod.rs @@ -60,9 +60,12 @@ const STACK_READY_POLL_INTERVAL: Duration = Duration::from_millis(250); const STACK_READY_REQUEST_TIMEOUT: Duration = Duration::from_secs(2); const CONFORMANCE_SERVER_ERA_ENV: &str = "CF_CONFORMANCE_SERVER_ERA"; const DEFAULT_CONFORMANCE_SERVER_ERA: ConformanceServerEra = ConformanceServerEra::Modern; +const LOAD_LOCUST_CPUSET_ENV: &str = "CF_LOAD_LOCUST_CPUSET"; +const LOAD_TARGET_CPUSET_ENV: &str = "CF_LOAD_TARGET_CPUSET"; mod ci; mod conformance; mod control_plane; +mod fyre; mod inspect; mod live; mod performance; @@ -108,6 +111,7 @@ impl RuntimeContext { .await } Action::Load(args) => self.run_load(args).await, + Action::Fyre(action) => self.execute_fyre(action).await, Action::Live { lane, group, diff --git a/src/runtime/performance/mod.rs b/src/runtime/performance/mod.rs index c990a63..913be98 100644 --- a/src/runtime/performance/mod.rs +++ b/src/runtime/performance/mod.rs @@ -2,6 +2,12 @@ use super::*; +#[derive(Debug, Clone, PartialEq, Eq)] +struct LoadCpuSplit { + target: String, + locust: String, +} + impl RuntimeContext { pub(super) async fn start_standalone_fast_time(&self, observability: bool) -> AppResult<()> { let command = self.standalone_dataplane_project(observability).command([ @@ -59,6 +65,10 @@ impl RuntimeContext { pub(super) async fn run_load(&self, args: ResolvedLoadArgs) -> AppResult<()> { let settings = LoadSettings::resolve(&self.config, &args.request).map_err(AppFailure::from)?; + let cpu_split = args + .isolate_cpus + .then(|| self.load_cpu_split()) + .transpose()?; let server_id = self.default_server_id().to_owned(); let operation_server_id = server_id.clone(); let preparation = Activity::spinner("Preparing performance stack"); @@ -69,6 +79,8 @@ impl RuntimeContext { args.standalone, args.observability, args.client_era, + args.builtin_memory_limit.clone(), + cpu_split.as_ref().map(|split| split.target.clone()), ), |token, standalone_tool_names| async move { let project = if args.standalone { @@ -97,18 +109,28 @@ impl RuntimeContext { .env("MCP_TOOL_NAMES", standalone_tool_names.join(",")) .env("MCP_SKIP_TOOL_LIST", "true"); } + if let Some(split) = &cpu_split { + command_spec = command_spec.env(LOAD_LOCUST_CPUSET_ENV, split.locust.as_str()); + } let output_log = command.report_dir().join("locust.log"); fs::write(&output_log, []) .with_context(|| format!("failed to clear Locust output log {output_log:?}")) .map_err(AppFailure::from)?; preparation.finish(true); - let description = format!( - "Running load test ({} users, {}/s, {})", + let mut description = format!( + "Running load test ({} users, {}/s, {}, {} workers)", settings.users(), settings.spawn_rate(), settings.run_time(), + settings.workers(), ); + if let Some(split) = &cpu_split { + description.push_str(&format!( + ", target CPUs {}, Locust CPUs {}", + split.target, split.locust + )); + } let activity = Activity::spinner(description); let started = std::time::Instant::now(); let process_result = self @@ -153,6 +175,39 @@ impl RuntimeContext { ) .await } + + fn load_cpu_split(&self) -> AppResult { + let value = + self.capture_text(&CommandSpec::new("docker").args(["info", "--format", "{{.NCPU}}"]))?; + let cpus = value.parse::().map_err(|_| { + AppFailure::from(anyhow!( + "Docker returned an invalid CPU count for isolation" + )) + })?; + split_load_cpus(cpus) + } +} + +fn split_load_cpus(cpus: usize) -> AppResult { + if cpus < 2 { + return Err(AppFailure::from(anyhow!( + "--isolate-cpus requires Docker to expose at least two CPUs" + ))); + } + let target_end = cpus / 2 - 1; + let locust_start = target_end + 1; + Ok(LoadCpuSplit { + target: cpu_range(0, target_end), + locust: cpu_range(locust_start, cpus - 1), + }) +} + +fn cpu_range(start: usize, end: usize) -> String { + if start == end { + start.to_string() + } else { + format!("{start}-{end}") + } } fn finalize_locust_run( @@ -163,3 +218,32 @@ fn finalize_locust_run( audit_locust_reports(report_dir, bearer_token).map_err(AppFailure::from)?; process_result } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn load_cpu_split_partitions_every_available_cpu() { + assert_eq!( + split_load_cpus(16).expect("sixteen CPUs should split"), + LoadCpuSplit { + target: String::from("0-7"), + locust: String::from("8-15"), + } + ); + assert_eq!( + split_load_cpus(3).expect("three CPUs should split"), + LoadCpuSplit { + target: String::from("0"), + locust: String::from("1-2"), + } + ); + assert_eq!( + split_load_cpus(1) + .expect_err("one CPU cannot be isolated") + .to_string(), + "--isolate-cpus requires Docker to expose at least two CPUs" + ); + } +} diff --git a/src/runtime/session.rs b/src/runtime/session.rs index 3e2fb9a..9c0d84f 100644 --- a/src/runtime/session.rs +++ b/src/runtime/session.rs @@ -44,6 +44,8 @@ pub(super) struct ManagedTargetOptions { observability: bool, load: bool, backend: StandaloneBackend, + builtin_memory_limit: Option, + load_target_cpuset: Option, } impl ManagedTargetOptions { @@ -57,6 +59,8 @@ impl ManagedTargetOptions { observability, load: false, backend: StandaloneBackend::Conformance(protocol_version), + builtin_memory_limit: None, + load_target_cpuset: None, } } @@ -64,12 +68,16 @@ impl ManagedTargetOptions { standalone: bool, observability: bool, protocol_version: ProtocolVersion, + builtin_memory_limit: Option, + load_target_cpuset: Option, ) -> Self { Self { standalone, observability, load: true, backend: StandaloneBackend::FastTime(protocol_version), + builtin_memory_limit, + load_target_cpuset, } } } @@ -127,14 +135,42 @@ impl RuntimeContext { STANDALONE_USER_ID, ]); let command = self.standalone_dataplane_environment(command, true)?; - let output = self.runner.capture_stdout(&command)?; + self.capture_harness_token(&command) + } + + pub(super) fn client_conformance_token( + &self, + standalone: bool, + ) -> AppResult { + let subject = format!("cf-integration-client-{}", uuid::Uuid::new_v4().simple()); + let project = if standalone { + self.standalone_conformance_compose_project(true) + } else { + self.conformance_runtime_project(StackMode::Dataplane) + }; + let command = project.command([ + "run", + "--quiet-build", + "--rm", + "--no-deps", + "config_writer", + "token", + STANDALONE_TENANT_ID, + &subject, + ]); + let command = self.target_environment(command, StackMode::Dataplane, standalone)?; + self.capture_harness_token(&command) + } + + fn capture_harness_token(&self, command: &CommandSpec) -> AppResult { + let output = self.runner.capture_stdout(command)?; let token = std::str::from_utf8(&output) - .context("standalone dataplane token helper returned non-UTF-8 output") + .context("dataplane token helper returned non-UTF-8 output") .map_err(AppFailure::from)? .trim(); if token.split('.').count() != 3 { return Err(AppFailure::from(anyhow!( - "standalone dataplane token helper returned an invalid JWT" + "dataplane token helper returned an invalid JWT" ))); } Ok(ManagedBearerToken::unmanaged(token.to_owned())) @@ -179,8 +215,12 @@ impl RuntimeContext { let mut scope = ManagedSessionScope::new(self, topology, options.standalone); let primary = async { let token = if options.standalone { - self.stack_up_standalone_dataplane(false, options.observability) - .await?; + self.stack_up_standalone_dataplane( + false, + options.observability, + options.load_target_cpuset.as_deref(), + ) + .await?; match options.backend { StandaloneBackend::Conformance(version) => { self.start_standalone_fixture(&version, options.observability) @@ -195,8 +235,18 @@ impl RuntimeContext { } else { let project = self.performance_compose_project(topology, options.observability, options.load); - self.stack_up_with_project(topology, false, project, false, options.observability) - .await?; + self.stack_up_with_project( + topology, + false, + project, + false, + options.observability, + stack::StackRuntimeOverrides { + builtin_memory_limit: options.builtin_memory_limit.as_deref(), + load_target_cpuset: options.load_target_cpuset.as_deref(), + }, + ) + .await?; self.prepare_test_target(topology, server_id).await?; self.managed_bearer_token(topology, server_id).await? }; diff --git a/src/runtime/stack/mod.rs b/src/runtime/stack/mod.rs index a32e0ff..8a477eb 100644 --- a/src/runtime/stack/mod.rs +++ b/src/runtime/stack/mod.rs @@ -6,6 +6,12 @@ use super::*; const COMPOSE_PROTOCOL_VERSION_ENV: &str = "MCP_PROTOCOL_VERSION"; +#[derive(Default)] +pub(super) struct StackRuntimeOverrides<'a> { + pub(super) builtin_memory_limit: Option<&'a str>, + pub(super) load_target_cpuset: Option<&'a str>, +} + impl RuntimeContext { pub(super) async fn execute_stack(&self, action: StackAction) -> AppResult<()> { match action { @@ -16,7 +22,8 @@ impl RuntimeContext { standalone, } => { if standalone { - self.stack_up_standalone_dataplane(fresh, true).await?; + self.stack_up_standalone_dataplane(fresh, true, None) + .await?; } else { self.stack_up_for_conformance(topology, fresh).await?; } @@ -151,8 +158,15 @@ impl RuntimeContext { } pub(super) async fn stack_up(&self, mode: StackMode, fresh: bool) -> AppResult<()> { - self.stack_up_with_project(mode, fresh, self.compose_project(mode), false, true) - .await + self.stack_up_with_project( + mode, + fresh, + self.compose_project(mode), + false, + true, + StackRuntimeOverrides::default(), + ) + .await } pub(super) async fn stack_up_for_conformance( @@ -166,6 +180,7 @@ impl RuntimeContext { self.conformance_runtime_project(mode), false, true, + StackRuntimeOverrides::default(), ) .await } @@ -174,6 +189,7 @@ impl RuntimeContext { &self, fresh: bool, observability: bool, + load_target_cpuset: Option<&str>, ) -> AppResult<()> { if !self.config.dataplane_ref().value.is_empty() { self.ensure_dataplane()?; @@ -214,7 +230,10 @@ impl RuntimeContext { let command = self .standalone_dataplane_project(observability) .command(arguments); - let command = self.standalone_dataplane_environment(command, true)?; + let mut command = self.standalone_dataplane_environment(command, true)?; + if let Some(cpuset) = load_target_cpuset { + command = command.env(LOAD_TARGET_CPUSET_ENV, cpuset); + } self.runner.run_async(&command).await?; self.wait_for_public_endpoint(StackMode::Dataplane, false) .await @@ -227,6 +246,7 @@ impl RuntimeContext { project: ComposeProject, report_progress: bool, observability: bool, + overrides: StackRuntimeOverrides<'_>, ) -> AppResult<()> { self.ensure_mode_sources(mode)?; if mode == StackMode::Dataplane { @@ -275,7 +295,13 @@ impl RuntimeContext { AppFailure::from(anyhow!("CONTROLPLANE_LOCUST_WORKERS must be an integer")) })?; let command = stack_up_command(project, mode, build, start_locust, locust_workers); - let command = self.compose_environment(command, mode, true)?; + let mut command = self.compose_environment(command, mode, true)?; + if let Some(limit) = overrides.builtin_memory_limit { + command = command.env("GATEWAY_MEM_LIMIT", limit); + } + if let Some(cpuset) = overrides.load_target_cpuset { + command = command.env(LOAD_TARGET_CPUSET_ENV, cpuset); + } let (controlplane_pull_policy, dataplane_pull_policy) = compose_pull_policies( mode, build,