Run the frozen requirement sets in conformance CI

.github/workflows/conformance.yml

@@ -27,3 +27,26 @@ jobs:
         with:
           node-version: '24' # Specify the latest Node.js version.
       - run: bundle exec rake conformance:test
+
+  # Frozen requirement sets for SEP-1730 tier scoring: each spec revision's scenarios run at
+  # that revision's own wire version, unlike the moving active suite above.
+  requirements-conformance:
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    name: requirements-conformance (${{ matrix.requirements }})
+    strategy:
+      fail-fast: false
+      matrix:
+        requirements: ['2025-11-25', '2026-07-28']
+    steps:
+      - uses: actions/checkout@v7
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: '4.0' # Specify the latest supported Ruby version.
+          bundler-cache: true
+      - uses: actions/setup-node@v7
+        with:
+          node-version: '24' # Specify the latest Node.js version.
+      - run: bundle exec rake conformance:test
+        env:
+          REQUIREMENTS: ${{ matrix.requirements }}

Rakefile

@@ -17,17 +17,22 @@ RuboCop::RakeTask.new
 task default: [:rubocop, :test, "conformance:test"]
 
 namespace :conformance do
-  desc "Run MCP conformance tests (PORT, SCENARIO, SPEC_VERSION, VERBOSE)"
+  desc "Run MCP conformance tests (PORT, SCENARIO, SPEC_VERSION, REQUIREMENTS, VERBOSE)"
   task :test do |t|
     next unless npx_available?(t.name)
 
+    if ENV["REQUIREMENTS"] && (ENV["SCENARIO"] || ENV["SPEC_VERSION"])
+      abort("REQUIREMENTS cannot be combined with SCENARIO or SPEC_VERSION: a requirement set already fixes which scenarios run.")
+    end
+
     require_relative "conformance/server_runner"
     require_relative "conformance/client_runner"
 
     options = {}
     options[:port] = Integer(ENV["PORT"]) if ENV["PORT"]
     options[:scenario] = ENV["SCENARIO"] if ENV["SCENARIO"]
     options[:spec_version] = ENV["SPEC_VERSION"] if ENV["SPEC_VERSION"]
+    options[:requirements] = ENV["REQUIREMENTS"] if ENV["REQUIREMENTS"]
     options[:verbose] = true if ENV["VERBOSE"]
 
     Conformance::ServerRunner.new(**options).run

conformance/README.md

@@ -22,17 +22,25 @@ are allowed to fail without affecting the exit code.
 
 ### Environment variables
 
-| Variable       | Description                           | Default |
-|----------------|---------------------------------------|---------|
-| `PORT`         | Server port (server conformance only) | `9292`  |
-| `SCENARIO`     | Run a single scenario by name         | (all)   |
-| `SPEC_VERSION` | Filter scenarios by spec version      | (all)   |
-| `VERBOSE`      | Show raw JSON output when set         | (off)   |
+| Variable       | Description                                                                  | Default |
+|----------------|------------------------------------------------------------------------------|---------|
+| `PORT`         | Server port (server conformance only)                                        | `9292`  |
+| `SCENARIO`     | Run a single scenario by name                                                | (all)   |
+| `SPEC_VERSION` | Filter scenarios by spec version                                             | (all)   |
+| `REQUIREMENTS` | Run a spec revision's frozen requirement set at that revision's wire version | (off)   |
+| `VERBOSE`      | Show raw JSON output when set                                                | (off)   |
+
+`REQUIREMENTS` runs exactly the scenarios a spec revision required when it shipped, which is
+what SEP-1730 tier scoring measures. It cannot be combined with `SCENARIO` or `SPEC_VERSION`:
+a requirement set already fixes which scenarios run.
 
 ```bash
 # Run a single server scenario
 bundle exec rake conformance:test SCENARIO=ping
 
+# Run the frozen 2026-07-28 requirement set
+bundle exec rake conformance:test REQUIREMENTS=2026-07-28
+
 # Use a different port with verbose output
 bundle exec rake conformance:test PORT=3000 VERBOSE=1
 

conformance/client_runner.rb

@@ -5,9 +5,10 @@
 
 module Conformance
   class ClientRunner
-    def initialize(scenario: nil, spec_version: nil, verbose: false)
+    def initialize(scenario: nil, spec_version: nil, requirements: nil, verbose: false)
       @scenario = scenario
       @spec_version = spec_version
+      @requirements = requirements
       @verbose = verbose
     end
 
@@ -34,7 +35,10 @@ def build_command
         "--command",
         "bundle exec ruby #{client_script}",
       ]
-      npx_command += if @scenario
+      # A requirement set already fixes which scenarios run, so it replaces the suite selection.
+      npx_command += if @requirements
+        ["--requirements", @requirements]
+      elsif @scenario
         ["--scenario", @scenario]
       else
         ["--suite", "all"]

conformance/server_runner.rb

@@ -12,10 +12,11 @@ class ServerRunner
     SERVER_POLL_INTERVAL = 0.5
     SERVER_HEALTH_CHECK_RETRIES = (SERVER_START_TIMEOUT / SERVER_POLL_INTERVAL).to_i
 
-    def initialize(port: Server::DEFAULT_PORT, scenario: nil, spec_version: nil, verbose: false)
+    def initialize(port: Server::DEFAULT_PORT, scenario: nil, spec_version: nil, requirements: nil, verbose: false)
       @port = port
       @scenario = scenario
       @spec_version = spec_version
+      @requirements = requirements
       @verbose = verbose
     end
 
@@ -34,6 +35,7 @@ def build_command
       npx_command = ["npx", "--yes", "@modelcontextprotocol/conformance@alpha", "server", "--url", "http://localhost:#{@port}/mcp"]
       npx_command += ["--scenario", @scenario] if @scenario
       npx_command += ["--spec-version", @spec_version] if @spec_version
+      npx_command += ["--requirements", @requirements] if @requirements
       npx_command += ["--verbose"] if @verbose
       npx_command += ["--expected-failures", expected_failures_yml]
       npx_command

test/conformance/client_runner_test.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+require "test_helper"
+require_relative "../../conformance/client_runner"
+
+module Conformance
+  class ClientRunnerTest < ActiveSupport::TestCase
+    EXPECTED_FAILURES_YML = File.expand_path("../../conformance/expected_failures.yml", __dir__)
+    CLIENT_SCRIPT = File.expand_path("../../conformance/client.rb", __dir__)
+
+    test "build_command runs the full suite by default" do
+      command = ClientRunner.new.send(:build_command)
+
+      assert_equal(
+        [
+          "npx",
+          "--yes",
+          "@modelcontextprotocol/conformance@alpha",
+          "client",
+          "--command",
+          "bundle exec ruby #{CLIENT_SCRIPT}",
+          "--suite",
+          "all",
+          "--expected-failures",
+          EXPECTED_FAILURES_YML,
+        ],
+        command,
+      )
+    end
+
+    test "build_command replaces the suite selection with a frozen requirement set" do
+      command = ClientRunner.new(requirements: "2026-07-28").send(:build_command)
+
+      assert_equal(
+        [
+          "npx",
+          "--yes",
+          "@modelcontextprotocol/conformance@alpha",
+          "client",
+          "--command",
+          "bundle exec ruby #{CLIENT_SCRIPT}",
+          "--requirements",
+          "2026-07-28",
+          "--expected-failures",
+          EXPECTED_FAILURES_YML,
+        ],
+        command,
+      )
+    end
+
+    test "build_command keeps the single scenario selection" do
+      command = ClientRunner.new(scenario: "tools_call").send(:build_command)
+
+      assert_equal(
+        [
+          "npx",
+          "--yes",
+          "@modelcontextprotocol/conformance@alpha",
+          "client",
+          "--command",
+          "bundle exec ruby #{CLIENT_SCRIPT}",
+          "--scenario",
+          "tools_call",
+          "--expected-failures",
+          EXPECTED_FAILURES_YML,
+        ],
+        command,
+      )
+    end
+  end
+end

test/conformance/server_runner_test.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+require "test_helper"
+require_relative "../../conformance/server_runner"
+
+module Conformance
+  class ServerRunnerTest < ActiveSupport::TestCase
+    EXPECTED_FAILURES_YML = File.expand_path("../../conformance/expected_failures.yml", __dir__)
+
+    test "build_command runs the active suite by default" do
+      command = ServerRunner.new.send(:build_command)
+
+      assert_equal(
+        [
+          "npx",
+          "--yes",
+          "@modelcontextprotocol/conformance@alpha",
+          "server",
+          "--url",
+          "http://localhost:#{Server::DEFAULT_PORT}/mcp",
+          "--expected-failures",
+          EXPECTED_FAILURES_YML,
+        ],
+        command,
+      )
+    end
+
+    test "build_command runs a frozen requirement set and keeps the baseline" do
+      command = ServerRunner.new(requirements: "2026-07-28").send(:build_command)
+
+      assert_equal(
+        [
+          "npx",
+          "--yes",
+          "@modelcontextprotocol/conformance@alpha",
+          "server",
+          "--url",
+          "http://localhost:#{Server::DEFAULT_PORT}/mcp",
+          "--requirements",
+          "2026-07-28",
+          "--expected-failures",
+          EXPECTED_FAILURES_YML,
+        ],
+        command,
+      )
+    end
+  end
+end