diff --git a/examples/servers/typescript/everything-server.ts b/examples/servers/typescript/everything-server.ts index d7de2228..342b001c 100644 --- a/examples/servers/typescript/everything-server.ts +++ b/examples/servers/typescript/everything-server.ts @@ -439,42 +439,75 @@ function createMcpServer() { inputSchema: {} // Empty schema so callback gets (args, extra) instead of just (extra) }, async (_args, { sendNotification, _meta }) => { - const progressToken = _meta?.progressToken ?? 0; - console.log('???? Progress token:', progressToken); - await sendNotification({ - method: 'notifications/progress', - params: { - progressToken, - progress: 0, - total: 100, - message: `Completed step ${0} of ${100}` - } - }); - await new Promise((resolve) => setTimeout(resolve, 50)); + const progressToken = _meta?.progressToken; + if (progressToken !== undefined) { + await sendNotification({ + method: 'notifications/progress', + params: { + progressToken, + progress: 0, + total: 100, + message: `Completed step ${0} of ${100}` + } + }); + await new Promise((resolve) => setTimeout(resolve, 50)); + + await sendNotification({ + method: 'notifications/progress', + params: { + progressToken, + progress: 50, + total: 100, + message: `Completed step ${50} of ${100}` + } + }); + await new Promise((resolve) => setTimeout(resolve, 50)); + + await sendNotification({ + method: 'notifications/progress', + params: { + progressToken, + progress: 100, + total: 100, + message: `Completed step ${100} of ${100}` + } + }); + } - await sendNotification({ - method: 'notifications/progress', - params: { - progressToken, - progress: 50, - total: 100, - message: `Completed step ${50} of ${100}` - } - }); - await new Promise((resolve) => setTimeout(resolve, 50)); + return { + content: [{ type: 'text', text: String(progressToken ?? 'no-token') }] + }; + } + ); - await sendNotification({ - method: 'notifications/progress', - params: { - progressToken, - progress: 100, - total: 100, - message: `Completed step ${100} of ${100}` - } - }); + // Slow tool for cancellation testing + mcpServer.registerTool( + 'test_tool_slow', + { + description: + 'Sleeps for the specified duration (durationMs, default 5000) before returning. Used for cancellation testing.', + inputSchema: {} + }, + async (args) => { + const duration = (args as { durationMs?: number }).durationMs ?? 5000; + await new Promise((resolve) => setTimeout(resolve, duration)); + return { + content: [{ type: 'text', text: `Slept for ${duration}ms` }] + }; + } + ); + // Fast tool for health-check after cancellation + mcpServer.registerTool( + 'test_tool_fast', + { + description: + 'Returns immediately. Used as a health check after cancellation tests.', + inputSchema: {} + }, + async () => { return { - content: [{ type: 'text', text: String(progressToken) }] + content: [{ type: 'text', text: 'Fast response' }] }; } ); diff --git a/src/scenarios/index.ts b/src/scenarios/index.ts index 03cf55f5..9952822f 100644 --- a/src/scenarios/index.ts +++ b/src/scenarios/index.ts @@ -66,6 +66,8 @@ import { PromptsGetWithImageScenario } from './server/prompts'; +import { CancellationScenario } from './server/cancellation'; +import { ProgressNotificationsScenario } from './server/progress-notifications'; import { DNSRebindingProtectionScenario } from './server/dns-rebinding'; import { CachingScenario } from './server/caching'; @@ -123,6 +125,12 @@ import { JsonSchema2020_12PreservationScenario } from './client/json-schema-2020 // Pending client scenarios (not yet fully tested/implemented) const pendingClientScenariosList: ClientScenario[] = [ + // Cancellation and progress scenarios require test_tool_slow, test_tool_fast, + // and the progressToken guard fix in the everything-server. These additions + // are included in this branch but not yet merged to the server's main. + new CancellationScenario(), + new ProgressNotificationsScenario(), + // JSON Schema 2020-12 (SEP-1613) // This test is pending until the SDK includes PR #1135 which preserves // $schema, $defs, and additionalProperties fields in tool schemas. @@ -165,6 +173,8 @@ const allClientScenariosList: ClientScenario[] = [ new LoggingSetLevelScenario(), new PingScenario(), new CompletionCompleteScenario(), + new CancellationScenario(), + new ProgressNotificationsScenario(), // Tools scenarios new ToolsListScenario(), diff --git a/src/scenarios/server/cancellation.ts b/src/scenarios/server/cancellation.ts new file mode 100644 index 00000000..206e02ea --- /dev/null +++ b/src/scenarios/server/cancellation.ts @@ -0,0 +1,259 @@ +/** + * Cancellation conformance test scenario for MCP servers. + * + * Validates that servers handle `notifications/cancelled` gracefully per spec: + * - MUST NOT crash or enter invalid state on cancellation of unknown requests + * - SHOULD stop processing cancelled in-progress requests + * - MUST remain stable under rapid cancellation bursts + * + * Closes https://github.com/modelcontextprotocol/conformance/issues/433 + */ + +import { ClientScenario, ConformanceCheck } from '../../types'; +import { type RunContext } from '../../connection'; +import { connectToServer } from '../../connection/sdk-client'; + +const SPEC_REFERENCES = [ + { + id: 'MCP-Cancellation', + url: 'https://modelcontextprotocol.io/specification/2025-06-18/basic/utilities/cancellation' + } +]; + +interface CheckDef { + id: string; + name: string; + description: string; +} + +const UNKNOWN_REQUEST_STABILITY: CheckDef = { + id: 'cancellation-unknown-request-stability', + name: 'CancellationUnknownRequestStability', + description: 'Server remains stable after cancellation of unknown request ID' +}; + +const IN_PROGRESS_REQUEST: CheckDef = { + id: 'cancellation-in-progress-request', + name: 'CancellationInProgressRequest', + description: + 'Server handles cancellation of in-progress request without degradation' +}; + +const RAPID_BURST_STABILITY: CheckDef = { + id: 'cancellation-rapid-burst-stability', + name: 'CancellationRapidBurstStability', + description: 'Server remains stable under rapid cancellation notifications' +}; + +function check( + def: CheckDef, + status: ConformanceCheck['status'], + extras: Pick, 'errorMessage' | 'details'> = {} +): ConformanceCheck { + return { + ...def, + status, + timestamp: new Date().toISOString(), + specReferences: SPEC_REFERENCES, + ...extras + }; +} + +export class CancellationScenario implements ClientScenario { + name = 'cancellation'; + readonly source = { introducedIn: '2025-06-18' } as const; + description = `Test cancellation notification handling. + +**Server Implementation Requirements:** + +**Notification**: \`notifications/cancelled\` + +**Requirements**: +- Server MUST handle cancellation gracefully without crashing or entering an invalid state +- Server SHOULD stop processing the cancelled request and free associated resources +- Server MAY ignore cancellation if the request is unknown, already completed, or not cancellable +- The \`notifications/cancelled\` params MUST include \`requestId\` corresponding to the ID of a previously issued request + +**Test Server Prerequisites:** +- Must expose a tool named \`test_tool_slow\` that accepts \`{ durationMs: number }\` and sleeps for that duration before returning +- Must expose a tool named \`test_tool_fast\` that completes immediately with a text response`; + + async run(ctx: RunContext): Promise { + const checks: ConformanceCheck[] = []; + const { serverUrl, specVersion } = ctx; + // Uses connectToServer() directly (not ctx.connect()) because the Connection + // interface does not expose a notification() method needed to send + // notifications/cancelled to the server. + + // Check 1: Server remains stable after receiving cancellation for unknown request + try { + const connection = await connectToServer(serverUrl, {}, specVersion); + + // Send cancellation notification for a request ID that was never issued. + // The SDK client.notification() routes through the managed transport, + // which includes session headers automatically. + await connection.client.notification({ + method: 'notifications/cancelled', + params: { + requestId: 'nonexistent-request-99999', + reason: 'Testing unknown request cancellation' + } + }); + + // Verify server is still responsive + const result = await connection.client.callTool({ + name: 'test_tool_fast', + arguments: {} + }); + + await connection.close(); + + if (!result || !result.content) { + checks.push( + check(UNKNOWN_REQUEST_STABILITY, 'FAILURE', { + errorMessage: + 'Server did not respond after receiving cancellation for unknown request' + }) + ); + } else { + checks.push( + check(UNKNOWN_REQUEST_STABILITY, 'SUCCESS', { + details: { serverResponded: true } + }) + ); + } + } catch (error) { + checks.push( + check(UNKNOWN_REQUEST_STABILITY, 'FAILURE', { + errorMessage: `Failed: ${error instanceof Error ? error.message : String(error)}` + }) + ); + } + + // Check 2: Cancellation of an in-progress request + try { + const connection = await connectToServer(serverUrl, {}, specVersion); + const startTime = Date.now(); + + // Start the slow tool call without awaiting completion + const slowPromise = connection.client.callTool({ + name: 'test_tool_slow', + arguments: { durationMs: 10000 } + }); + + // Wait for the server to begin processing + await new Promise((resolve) => setTimeout(resolve, 1000)); + + // Send cancellation. The SDK assigns sequential numeric request IDs; + // after initialize (id=0), the first callTool gets id=1. + await connection.client.notification({ + method: 'notifications/cancelled', + params: { + requestId: 1, + reason: 'Client no longer needs this result' + } + }); + + // Wait for the slow request to resolve, error, or timeout + let timedOut = false; + try { + await Promise.race([ + slowPromise, + new Promise((_, reject) => + setTimeout(() => reject(new Error('timeout')), 12000) + ) + ]); + } catch (e) { + if (e instanceof Error && e.message === 'timeout') { + timedOut = true; + } + // Other errors (e.g. request cancelled) are acceptable + } + + const elapsed = Date.now() - startTime; + + // Verify server is still healthy after the cancellation exchange + const healthCheck = await connection.client.callTool({ + name: 'test_tool_fast', + arguments: {} + }); + + await connection.close(); + + if (!healthCheck || !healthCheck.content) { + checks.push( + check(IN_PROGRESS_REQUEST, 'FAILURE', { + errorMessage: + 'Server became unresponsive after cancellation of in-progress request', + details: { elapsedMs: elapsed, timedOut } + }) + ); + } else { + checks.push( + check(IN_PROGRESS_REQUEST, 'SUCCESS', { + details: { + elapsedMs: elapsed, + cancelledEarly: elapsed < 9000, + timedOut, + healthCheckPassed: true + } + }) + ); + } + } catch (error) { + checks.push( + check(IN_PROGRESS_REQUEST, 'FAILURE', { + errorMessage: `Failed: ${error instanceof Error ? error.message : String(error)}` + }) + ); + } + + // Check 3: Multiple rapid cancellations do not crash the server + try { + const connection = await connectToServer(serverUrl, {}, specVersion); + + // Fire several cancellation notifications for nonexistent requests. + // Server MAY ignore these but MUST remain stable. + for (let i = 0; i < 5; i++) { + await connection.client.notification({ + method: 'notifications/cancelled', + params: { + requestId: `burst-cancel-${i}`, + reason: 'Rapid cancellation burst test' + } + }); + } + + // Verify server is still responsive + const result = await connection.client.callTool({ + name: 'test_tool_fast', + arguments: {} + }); + + await connection.close(); + + if (!result || !result.content) { + checks.push( + check(RAPID_BURST_STABILITY, 'FAILURE', { + errorMessage: + 'Server became unresponsive after rapid cancellation burst' + }) + ); + } else { + checks.push( + check(RAPID_BURST_STABILITY, 'SUCCESS', { + details: { cancellationCount: 5, serverResponded: true } + }) + ); + } + } catch (error) { + checks.push( + check(RAPID_BURST_STABILITY, 'FAILURE', { + errorMessage: `Failed: ${error instanceof Error ? error.message : String(error)}` + }) + ); + } + + return checks; + } +} diff --git a/src/scenarios/server/progress-notifications.ts b/src/scenarios/server/progress-notifications.ts new file mode 100644 index 00000000..377a8842 --- /dev/null +++ b/src/scenarios/server/progress-notifications.ts @@ -0,0 +1,334 @@ +/** + * Progress notification edge-case conformance test scenario for MCP servers. + * + * Extends coverage beyond the basic progress check in tools.ts by validating: + * - Non-decreasing progress values + * - Token matching between request and notifications + * - No spurious notifications when no progressToken is provided + * - Cessation of notifications after request completion + * + * Closes https://github.com/modelcontextprotocol/conformance/issues/434 + */ + +import { ClientScenario, ConformanceCheck } from '../../types'; +import type { RunContext } from '../../connection'; +import type { CallToolResult } from '../../spec-types/2025-06-18'; + +const SPEC_REFERENCES = [ + { + id: 'MCP-Progress', + url: 'https://modelcontextprotocol.io/specification/2025-06-18/basic/utilities/progress' + } +]; + +interface CheckDef { + id: string; + name: string; + description: string; +} + +const VALUES_NON_DECREASING: CheckDef = { + id: 'progress-values-non-decreasing', + name: 'ProgressValuesNonDecreasing', + description: 'Progress values are non-decreasing and total is consistent' +}; + +const TOKEN_MATCHES_REQUEST: CheckDef = { + id: 'progress-token-matches-request', + name: 'ProgressTokenMatchesRequest', + description: + 'Progress notifications reference the correct token from the request' +}; + +const NO_TOKEN_NO_NOTIFICATIONS: CheckDef = { + id: 'progress-no-token-no-notifications', + name: 'ProgressNoTokenNoNotifications', + description: + 'Server does not send progress notifications when no progressToken is provided' +}; + +const CEASES_AFTER_COMPLETION: CheckDef = { + id: 'progress-ceases-after-completion', + name: 'ProgressCeasesAfterCompletion', + description: 'No progress notifications arrive after the request response' +}; + +function check( + def: CheckDef, + status: ConformanceCheck['status'], + extras: Pick, 'errorMessage' | 'details'> = {} +): ConformanceCheck { + return { + ...def, + status, + timestamp: new Date().toISOString(), + specReferences: SPEC_REFERENCES, + ...extras + }; +} + +interface ProgressParams { + progressToken?: string | number; + progress: number; + total?: number; +} + +export class ProgressNotificationsScenario implements ClientScenario { + name = 'progress-notifications'; + readonly source = { introducedIn: '2025-06-18' } as const; + description = `Test progress notification semantics and constraints. + +**Server Implementation Requirements:** + +**Notification**: \`notifications/progress\` + +**Requirements**: +- Progress tokens MUST be string or integer +- The \`progress\` value MUST be non-decreasing across notifications for the same token +- Progress notifications MUST reference tokens from active requests only +- Progress notifications MUST cease after the request completes +- If \`total\` is provided, it SHOULD remain consistent or increase +- \`progress\` SHOULD NOT exceed \`total\` when total is provided +- Server MAY choose not to send progress notifications at all + +**Test Server Prerequisites:** +- Must expose \`test_tool_with_progress\` that emits at least 3 progress notifications when a progressToken is provided`; + + async run(ctx: RunContext): Promise { + const checks: ConformanceCheck[] = []; + + // Check 1: Progress values are non-decreasing + try { + const conn = await ctx.connect(); + await conn.request('tools/call', { + name: 'test_tool_with_progress', + arguments: {}, + _meta: { progressToken: 'progress-ordering-test' } + }); + + const progressUpdates = conn.notifications + .filter((n) => n.method === 'notifications/progress') + .map((n) => n.params as ProgressParams) + .filter((p) => p.progressToken === 'progress-ordering-test'); + + await conn.close(); + + if (progressUpdates.length === 0) { + checks.push( + check(VALUES_NON_DECREASING, 'INFO', { + errorMessage: + 'No progress notifications received. Server MAY choose not to send them.', + details: { progressCount: 0 } + }) + ); + } else { + const errors: string[] = []; + const warnings: string[] = []; + + for (let i = 1; i < progressUpdates.length; i++) { + if (progressUpdates[i].progress < progressUpdates[i - 1].progress) { + errors.push( + `Progress decreased: ${progressUpdates[i - 1].progress} -> ${progressUpdates[i].progress} at index ${i}` + ); + break; + } + } + + const totals = progressUpdates + .filter((p) => p.total !== undefined) + .map((p) => p.total as number); + if (totals.length > 1) { + for (let i = 1; i < totals.length; i++) { + if (totals[i] < totals[i - 1]) { + warnings.push( + `Total decreased: ${totals[i - 1]} -> ${totals[i]} at index ${i}` + ); + break; + } + } + } + + for (const p of progressUpdates) { + if (p.total !== undefined && p.progress > p.total) { + warnings.push( + `Progress (${p.progress}) exceeds total (${p.total})` + ); + break; + } + } + + const status = + errors.length > 0 + ? 'FAILURE' + : warnings.length > 0 + ? 'WARNING' + : 'SUCCESS'; + + checks.push( + check(VALUES_NON_DECREASING, status, { + errorMessage: errors.length > 0 ? errors.join('; ') : undefined, + details: { + progressCount: progressUpdates.length, + warnings: warnings.length > 0 ? warnings : undefined, + values: progressUpdates.map((p) => ({ + progress: p.progress, + total: p.total + })) + } + }) + ); + } + } catch (error) { + checks.push( + check(VALUES_NON_DECREASING, 'FAILURE', { + errorMessage: `Failed: ${error instanceof Error ? error.message : String(error)}` + }) + ); + } + + // Check 2: Progress token in notifications matches the one sent in _meta + try { + const conn = await ctx.connect(); + const token = 'unique-token-match-test-42'; + await conn.request('tools/call', { + name: 'test_tool_with_progress', + arguments: {}, + _meta: { progressToken: token } + }); + + const progressUpdates = conn.notifications + .filter((n) => n.method === 'notifications/progress') + .map((n) => n.params as ProgressParams); + + await conn.close(); + + if (progressUpdates.length === 0) { + checks.push( + check(TOKEN_MATCHES_REQUEST, 'INFO', { + errorMessage: + 'No progress notifications received; token matching cannot be validated.', + details: { expectedToken: token } + }) + ); + } else { + const mismatched = progressUpdates.filter( + (p) => p.progressToken !== token + ); + if (mismatched.length > 0) { + checks.push( + check(TOKEN_MATCHES_REQUEST, 'FAILURE', { + errorMessage: `Received notifications with wrong token: expected "${token}", got "${mismatched[0].progressToken}"`, + details: { + expectedToken: token, + receivedTokens: progressUpdates.map((p) => p.progressToken) + } + }) + ); + } else { + checks.push( + check(TOKEN_MATCHES_REQUEST, 'SUCCESS', { + details: { + expectedToken: token, + notificationCount: progressUpdates.length + } + }) + ); + } + } + } catch (error) { + checks.push( + check(TOKEN_MATCHES_REQUEST, 'FAILURE', { + errorMessage: `Failed: ${error instanceof Error ? error.message : String(error)}` + }) + ); + } + + // Check 3: No progress notifications for requests without progressToken + try { + const conn = await ctx.connect(); + await conn.request('tools/call', { + name: 'test_tool_with_progress', + arguments: {} + }); + + const progressUpdates = conn.notifications.filter( + (n) => n.method === 'notifications/progress' + ); + + await conn.close(); + + if (progressUpdates.length > 0) { + checks.push( + check(NO_TOKEN_NO_NOTIFICATIONS, 'FAILURE', { + errorMessage: `Received ${progressUpdates.length} progress notifications for a request without progressToken`, + details: { unexpectedCount: progressUpdates.length } + }) + ); + } else { + checks.push( + check(NO_TOKEN_NO_NOTIFICATIONS, 'SUCCESS', { + details: { unexpectedCount: 0 } + }) + ); + } + } catch (error) { + checks.push( + check(NO_TOKEN_NO_NOTIFICATIONS, 'FAILURE', { + errorMessage: `Failed: ${error instanceof Error ? error.message : String(error)}` + }) + ); + } + + // Check 4: Progress notifications cease after request completes + try { + const conn = await ctx.connect(); + const token = 'cessation-test-token'; + await conn.request('tools/call', { + name: 'test_tool_with_progress', + arguments: {}, + _meta: { progressToken: token } + }); + + const countAtCompletion = conn.notifications + .filter((n) => n.method === 'notifications/progress') + .filter( + (n) => (n.params as ProgressParams)?.progressToken === token + ).length; + + // Wait briefly to catch any late notifications + await new Promise((resolve) => setTimeout(resolve, 200)); + + const countAfterWait = conn.notifications + .filter((n) => n.method === 'notifications/progress') + .filter( + (n) => (n.params as ProgressParams)?.progressToken === token + ).length; + + await conn.close(); + + const lateCount = countAfterWait - countAtCompletion; + if (lateCount > 0) { + checks.push( + check(CEASES_AFTER_COMPLETION, 'FAILURE', { + errorMessage: `Received ${lateCount} progress notifications after request completed`, + details: { countAtCompletion, countAfterWait, lateCount } + }) + ); + } else { + checks.push( + check(CEASES_AFTER_COMPLETION, 'SUCCESS', { + details: { countAtCompletion, lateNotifications: 0 } + }) + ); + } + } catch (error) { + checks.push( + check(CEASES_AFTER_COMPLETION, 'FAILURE', { + errorMessage: `Failed: ${error instanceof Error ? error.message : String(error)}` + }) + ); + } + + return checks; + } +}