diff --git a/.github/actions/conformance/expected-failures.yml b/.github/actions/conformance/expected-failures.yml new file mode 100644 index 0000000000..d6cdd0f381 --- /dev/null +++ b/.github/actions/conformance/expected-failures.yml @@ -0,0 +1,52 @@ +# Conformance scenarios not yet passing against the Python SDK on main. +# CI exits 0 if only these fail, exits 1 on unexpected failures or stale entries. +# +# Baseline established against @modelcontextprotocol/conformance pinned in +# .github/workflows/conformance.yml (CONFORMANCE_VERSION = 0.2.0-alpha.3). +# New conformance releases are adopted by deliberately bumping that pin and +# reconciling this file in the same change. +# +# Entries are grouped by SEP. As each SEP lands in the SDK the corresponding +# scenarios start passing and MUST be removed from this list (the runner fails +# on stale entries), so the baseline burns down per milestone. + +client: + # --- Draft-spec scenarios (in `--suite draft`, also part of `--suite all`) --- + # SEP-2575 (request metadata / _meta envelope): client does not populate the + # _meta envelope or the MCP-Protocol-Version header semantics yet. + - request-metadata + # SEP-2322 (multi-round-trip requests): client does not echo requestState / + # handle IncompleteResult yet. + - sep-2322-client-request-state + # SEP-2243 (HTTP standardization): no fixture handler / client header support yet. + - http-custom-headers + - http-invalid-tool-headers + # SEP-2106 (JSON Schema $ref handling): client still dereferences network $refs. + - json-schema-ref-no-deref + # SEP-2468 (authorization response iss parameter): not implemented in the client. + - auth/iss-supported + - auth/iss-not-advertised + - auth/iss-supported-missing + - auth/iss-wrong-issuer + - auth/iss-unexpected + - auth/iss-normalized + - auth/metadata-issuer-mismatch + # SEP-2352 (authorization server migration): client does not re-register when + # PRM authorization_servers changes. + - auth/authorization-server-migration + # SEP-837 (application_type during DCR): the check only fires on draft-version + # runs; this draft scenario is the one place the client still hits it. + - auth/offline-access-not-supported + + # --- Pre-existing scenarios that fail on checks added after conformance 0.1.15 --- + # SEP-2350 (scope step-up): WARNING-only; the expected-failures evaluator + # counts WARNINGs as failures. + - auth/scope-step-up + # SEP-990 (enterprise-managed authorization extension): no fixture handler / + # client support for the token-exchange + JWT bearer flow. + - auth/enterprise-managed-authorization + +# The `active` suite (30 scenarios / 42 assertions) is fully green against +# mcp-everything-server. Draft-suite entries are added when the workflow +# gains a `--suite draft` step. +server: [] diff --git a/.github/actions/conformance/run-server.sh b/.github/actions/conformance/run-server.sh index 01af136120..30068c18c8 100755 --- a/.github/actions/conformance/run-server.sh +++ b/.github/actions/conformance/run-server.sh @@ -7,15 +7,36 @@ SERVER_URL="http://localhost:${PORT}/mcp" SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" cd "$SCRIPT_DIR/../../.." -# Start everything-server +# Refuse to start if something is already listening on the port. The readiness +# check below cannot tell our server apart from a stale one, so a leftover +# listener would mean silently running conformance against old code. +if (: > "/dev/tcp/localhost/${PORT}") 2>/dev/null; then + echo "Error: port ${PORT} is already in use." >&2 + echo "Stop the stale process first (lsof -ti:${PORT} -sTCP:LISTEN | xargs kill) or set PORT to a free port." >&2 + exit 1 +fi + +echo "Starting mcp-everything-server on port ${PORT}..." uv run --frozen mcp-everything-server --port "$PORT" & SERVER_PID=$! -trap "kill $SERVER_PID 2>/dev/null || true; wait $SERVER_PID 2>/dev/null || true" EXIT -# Wait for server to be ready +cleanup() { + echo "Stopping server (PID: ${SERVER_PID})..." + kill $SERVER_PID 2>/dev/null || true + wait $SERVER_PID 2>/dev/null || true +} +trap cleanup EXIT + +# Wait for server to be ready. --max-time keeps a hung listener from wedging +# the loop, and a dead server process fails fast instead of retrying. +echo "Waiting for server to be ready..." MAX_RETRIES=30 RETRY_COUNT=0 -while ! curl -s "$SERVER_URL" > /dev/null 2>&1; do +while ! curl -s --max-time 2 "$SERVER_URL" > /dev/null 2>&1; do + if ! kill -0 $SERVER_PID 2>/dev/null; then + echo "Server process exited unexpectedly" >&2 + exit 1 + fi RETRY_COUNT=$((RETRY_COUNT + 1)) if [ $RETRY_COUNT -ge $MAX_RETRIES ]; then echo "Server failed to start after ${MAX_RETRIES} retries" >&2 @@ -26,5 +47,5 @@ done echo "Server ready at $SERVER_URL" -# Run conformance tests -npx @modelcontextprotocol/conformance@0.1.10 server --url "$SERVER_URL" "$@" +npx --yes @modelcontextprotocol/conformance@"${CONFORMANCE_VERSION:?set CONFORMANCE_VERSION (pinned in .github/workflows/conformance.yml)}" \ + server --url "$SERVER_URL" "$@" diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index 9c33d2936b..a35f1a4daf 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -13,10 +13,14 @@ concurrency: permissions: contents: read +env: + # Pinned conformance harness version. Bump deliberately and reconcile + # .github/actions/conformance/expected-failures.yml in the same change. + CONFORMANCE_VERSION: "0.2.0-alpha.3" + jobs: server-conformance: runs-on: ubuntu-latest - continue-on-error: true steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 with: @@ -29,11 +33,14 @@ jobs: with: node-version: 24 - run: uv sync --frozen --all-extras --package mcp-everything-server - - run: ./.github/actions/conformance/run-server.sh + - name: Run server conformance (active suite) + run: >- + ./.github/actions/conformance/run-server.sh + --suite active + --expected-failures ./.github/actions/conformance/expected-failures.yml client-conformance: runs-on: ubuntu-latest - continue-on-error: true steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 with: @@ -46,4 +53,9 @@ jobs: with: node-version: 24 - run: uv sync --frozen --all-extras --package mcp - - run: npx @modelcontextprotocol/conformance@0.1.13 client --command 'uv run --frozen python .github/actions/conformance/client.py' --suite all + - name: Run client conformance (all suite) + run: >- + npx --yes @modelcontextprotocol/conformance@"$CONFORMANCE_VERSION" client + --command 'uv run --frozen python .github/actions/conformance/client.py' + --suite all + --expected-failures ./.github/actions/conformance/expected-failures.yml