From eddd6374350fdbe58e823f86a6a704ad7f1b4512 Mon Sep 17 00:00:00 2001 From: Max Isbey <224885523+maxisbey@users.noreply.github.com> Date: Mon, 15 Jun 2026 15:25:59 +0000 Subject: [PATCH] ci(conformance): run server --suite draft and baseline the 2026-07-28 scenarios Adds the draft-spec server step on top of #2877, mirroring typescript-sdk's two-step server job (active then draft). - Second run-server.sh invocation with --suite draft. - expected-failures.yml server: populated with the 17 draft scenarios that fail or warn (14 FAILURE + 3 WARNING-only), grouped by SEP. Same 17-entry set as typescript-sdk's server baseline. - Two of the 19 draft scenarios are intentionally not baselined: their negative-case assertions are vacuously satisfied today by the stateful server's -32600 response. Noted in a trailing comment. client: section unchanged; --suite all already covers draft client scenarios via the PR #2877 baseline. --- .../actions/conformance/expected-failures.yml | 42 +++++++++++++++++-- .github/workflows/conformance.yml | 5 +++ 2 files changed, 43 insertions(+), 4 deletions(-) diff --git a/.github/actions/conformance/expected-failures.yml b/.github/actions/conformance/expected-failures.yml index d6cdd0f38..b27a83306 100644 --- a/.github/actions/conformance/expected-failures.yml +++ b/.github/actions/conformance/expected-failures.yml @@ -46,7 +46,41 @@ client: # client support for the token-exchange + JWT bearer flow. - auth/enterprise-managed-authorization -# The `active` suite (30 scenarios / 42 assertions) is fully green against -# mcp-everything-server. Draft-suite entries are added when the workflow -# gains a `--suite draft` step. -server: [] +server: + # --- Draft-spec scenarios (in `--suite draft`; the `active` suite is green) --- + # SEP-2575 (stateless HTTP / _meta envelope): server has no stateless mode, + # _meta-derived capabilities, error-code mappings, or server/discover yet. + - server-stateless + # SEP-2322 (multi-round-trip requests / IncompleteResult): not implemented; + # most scenarios currently fail early with "Missing session ID" because + # mcp-everything-server only runs in stateful mode. + - input-required-result-basic-elicitation + - input-required-result-basic-sampling + - input-required-result-basic-list-roots + - input-required-result-request-state + - input-required-result-multiple-input-requests + - input-required-result-multi-round + - input-required-result-non-tool-request + - input-required-result-result-type + - input-required-result-tampered-state + - input-required-result-capability-check + # SEP-2549 (caching): no ttlMs/cacheScope support; scenario also hits the + # stateful-mode "Missing session ID" error. + - caching + # SEP-2243 (HTTP header standardization): -32001 HeaderMismatch handling and + # case-insensitive/whitespace-trimmed header validation not implemented. + - http-header-validation + - http-custom-header-server-validation + # WARNING-only entries: these scenarios emit no FAILURE checks, only SHOULD-level + # WARNINGs, but the expected-failures evaluator counts WARNINGs as failures. + # SEP-2164: server returns -32600 (not -32602) and omits error.data.uri. + - sep-2164-resource-not-found + # SEP-2322 SHOULD-level behaviours (re-request missing inputResponses, ignore + # unrecognized inputResponses keys). + - input-required-result-missing-input-response + - input-required-result-ignore-extra-params + # Intentionally NOT baselined (2 of 19 draft scenarios): the SEP-2322 + # negative-case scenarios input-required-result-unsupported-methods and + # input-required-result-validate-input pass today only because the stateful + # server's -32600 "Missing session ID" satisfies their assertions. They will + # start failing for real once stateless mode lands; add them then. diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index a35f1a4da..d52ad46d1 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -38,6 +38,11 @@ jobs: ./.github/actions/conformance/run-server.sh --suite active --expected-failures ./.github/actions/conformance/expected-failures.yml + - name: Run server conformance (draft suite) + run: >- + ./.github/actions/conformance/run-server.sh + --suite draft + --expected-failures ./.github/actions/conformance/expected-failures.yml client-conformance: runs-on: ubuntu-latest