diff --git a/core/src/main/kotlin/org/evomaster/core/Main.kt b/core/src/main/kotlin/org/evomaster/core/Main.kt index d409ec8b91..96e093ac14 100644 --- a/core/src/main/kotlin/org/evomaster/core/Main.kt +++ b/core/src/main/kotlin/org/evomaster/core/Main.kt @@ -25,6 +25,8 @@ import org.evomaster.core.problem.externalservice.httpws.service.HttpWsExternalS import org.evomaster.core.problem.graphql.GraphQLIndividual import org.evomaster.core.problem.graphql.service.GraphQLBlackBoxModule import org.evomaster.core.problem.graphql.service.GraphQLModule +import org.evomaster.core.problem.mcp.McpIndividual +import org.evomaster.core.problem.mcp.service.McpBlackBoxModule import org.evomaster.core.problem.rest.data.RestIndividual import org.evomaster.core.problem.rest.service.* import org.evomaster.core.problem.rest.service.module.BlackBoxRestModule @@ -615,7 +617,10 @@ class Main { } EMConfig.ProblemType.MCP -> { - throw IllegalStateException("MCP server analysis is not yet supported") + if (!config.blackBox) { + throw IllegalStateException("MCP only supports black-box mode") + } + McpBlackBoxModule(false) } //this should never happen, unless we add new type and forget to add it here @@ -825,6 +830,28 @@ class Main { } } + private fun getAlgorithmKeyMcp(config: EMConfig): Key> { + + return when (config.algorithm) { + EMConfig.Algorithm.RANDOM -> + Key.get(object : TypeLiteral>() {}) + + EMConfig.Algorithm.MIO -> + Key.get(object : TypeLiteral>() {}) + + EMConfig.Algorithm.MOSA -> + Key.get(object : TypeLiteral>() {}) + + EMConfig.Algorithm.WTS -> + Key.get(object : TypeLiteral>() {}) + + EMConfig.Algorithm.SMARTS -> + Key.get(object : TypeLiteral>() {}) + + else -> throw IllegalStateException("Unrecognized algorithm ${config.algorithm} for MCP") + } + } + private fun getAlgorithmKeyRest(config: EMConfig): Key> { return when (config.algorithm) { @@ -896,6 +923,7 @@ class Main { EMConfig.ProblemType.GRAPHQL -> getAlgorithmKeyGraphQL(config) EMConfig.ProblemType.RPC -> getAlgorithmKeyRPC(config) EMConfig.ProblemType.WEBFRONTEND -> getAlgorithmKeyWeb(config) + EMConfig.ProblemType.MCP -> getAlgorithmKeyMcp(config) else -> throw IllegalStateException("Unrecognized problem type ${config.problemType}") } diff --git a/core/src/main/kotlin/org/evomaster/core/problem/mcp/McpAction.kt b/core/src/main/kotlin/org/evomaster/core/problem/mcp/McpAction.kt index 8527e8f5e5..a9a10b1026 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/mcp/McpAction.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/mcp/McpAction.kt @@ -4,6 +4,15 @@ import org.evomaster.core.problem.api.param.Param import org.evomaster.core.search.action.MainAction import org.evomaster.core.search.gene.Gene +/** + * Base class for all actions that can be executed against an MCP server. + * + * An action can be either a tool call ([McpToolCallAction]) or a resource read ([McpResourceReadAction]). + * + * @param id stable identifier used as the key in the action cluster + * and as the coverage target name (e.g. "tool:myTool", "resource:file:///data") + * @param parameters the mutable action's inputs + */ abstract class McpAction( val id: String, parameters: MutableList diff --git a/core/src/main/kotlin/org/evomaster/core/problem/mcp/McpCallResult.kt b/core/src/main/kotlin/org/evomaster/core/problem/mcp/McpCallResult.kt index 60327de838..1588e55805 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/mcp/McpCallResult.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/mcp/McpCallResult.kt @@ -2,6 +2,12 @@ package org.evomaster.core.problem.mcp import org.evomaster.core.search.action.ActionResult +/** + * Stores the outcome of executing a single [McpAction] during fitness evaluation. + * + * A result is considered an error when the MCP server sets `isError: true` in the tool-call + * response, or when the fitness function catches an exception from the server. + */ class McpCallResult : ActionResult { companion object { @@ -10,6 +16,10 @@ class McpCallResult : ActionResult { constructor(sourceLocalId: String) : super(sourceLocalId) + /** + * Copy constructor. Delegates to the [ActionResult] copy constructor + * which propagates [stopping], [deathSentence], and the results map. + */ private constructor(other: McpCallResult) : super(other) override fun copy(): McpCallResult = McpCallResult(this) diff --git a/core/src/main/kotlin/org/evomaster/core/problem/mcp/McpConst.kt b/core/src/main/kotlin/org/evomaster/core/problem/mcp/McpConst.kt new file mode 100644 index 0000000000..f386f1ec03 --- /dev/null +++ b/core/src/main/kotlin/org/evomaster/core/problem/mcp/McpConst.kt @@ -0,0 +1,20 @@ +package org.evomaster.core.problem.mcp + +object McpConst { + + /** + * JSON-RPC version used for all MCP messages, as required by the MCP specification. + */ + const val JSONRPC_VERSION = "2.0" + + /** + * MCP protocol version negotiated during the initialize handshake. + */ + const val PROTOCOL_VERSION = "2025-11-25" + + /** + * HTTP header used by the Streamable HTTP transport to carry the session id + * returned by the server after a successful initialize handshake. + */ + const val SESSION_ID_HEADER = "Mcp-Session-Id" +} diff --git a/core/src/main/kotlin/org/evomaster/core/problem/mcp/McpIndividual.kt b/core/src/main/kotlin/org/evomaster/core/problem/mcp/McpIndividual.kt index 740f77c9ec..e8052ec9df 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/mcp/McpIndividual.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/mcp/McpIndividual.kt @@ -8,6 +8,16 @@ import org.evomaster.core.search.Individual import org.evomaster.core.search.StructuralElement import org.evomaster.core.search.action.ActionComponent +/** + * An individual (test case) for MCP blackbox testing. + * + * Represents a sequence of [McpAction]s — tool calls and resource reads — to be executed + * against an MCP server in order. Each action is wrapped in an [EnterpriseActionGroup] and + * stored in the MAIN group. + * + * The search engine samples, mutates, and evaluates instances of this class via + * [McpSampler] and [McpBlackBoxFitness] respectively. + */ class McpIndividual( sampleType: SampleType, allActions: MutableList, diff --git a/core/src/main/kotlin/org/evomaster/core/problem/mcp/McpInputParam.kt b/core/src/main/kotlin/org/evomaster/core/problem/mcp/McpInputParam.kt index 035ab3d118..a28cc2eb1f 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/mcp/McpInputParam.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/mcp/McpInputParam.kt @@ -3,6 +3,7 @@ package org.evomaster.core.problem.mcp import org.evomaster.core.problem.api.param.Param import org.evomaster.core.search.gene.Gene +/** Parameter that carries the payload of a [McpToolCallAction]. */ class McpInputParam(name: String, gene: Gene) : Param(name, gene) { override fun copyContent(): Param = McpInputParam(name, gene.copy()) } diff --git a/core/src/main/kotlin/org/evomaster/core/problem/mcp/McpResourceReadAction.kt b/core/src/main/kotlin/org/evomaster/core/problem/mcp/McpResourceReadAction.kt index 1ec9c4246b..960e1e789c 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/mcp/McpResourceReadAction.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/mcp/McpResourceReadAction.kt @@ -3,11 +3,11 @@ package org.evomaster.core.problem.mcp import org.evomaster.core.search.action.Action /** - * Action to read an MCP resource. - * For direct resources (isTemplate=false), the URI is fixed and no genes are mutable. - * For template resources (isTemplate=true), one StringGene per URI template variable - * (e.g. {city} in weather:///{city}/current) is created and fuzzed independently; - * call resolvedUri() to interpolate current gene values into the template. + * Action that reads a resource from an MCP server via the `resources/read` JSON-RPC method. + * + * @param uriTemplate the resource URI or URI template string (e.g. `file:///data`, `weather:///{city}/current`) + * @param uriParams one [McpUriParam] per template variable; empty for direct resources + * @param isTemplate whether this action represents a URI template or a fixed URI */ class McpResourceReadAction( val uriTemplate: String, diff --git a/core/src/main/kotlin/org/evomaster/core/problem/mcp/McpToolCallAction.kt b/core/src/main/kotlin/org/evomaster/core/problem/mcp/McpToolCallAction.kt index bd4ebdda1a..481a483029 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/mcp/McpToolCallAction.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/mcp/McpToolCallAction.kt @@ -3,15 +3,20 @@ package org.evomaster.core.problem.mcp import org.evomaster.core.search.action.Action import org.evomaster.core.search.gene.ObjectGene +/** + * Action that invokes a named tool on an MCP server via the `tools/call` JSON-RPC method. + * + * @param toolName the name of the MCP tool to call + * @param inputSchema an [ObjectGene] which fields represent the tool's input arguments + */ class McpToolCallAction( val toolName: String, - val inputSchema: ObjectGene, - val description: String = "" + val inputSchema: ObjectGene ) : McpAction( id = "tool:$toolName", parameters = mutableListOf(McpInputParam("input", inputSchema)) ) { override fun copyContent(): Action { - return McpToolCallAction(toolName, inputSchema.copy() as ObjectGene, description) + return McpToolCallAction(toolName, inputSchema.copy() as ObjectGene) } } diff --git a/core/src/main/kotlin/org/evomaster/core/problem/mcp/McpUriParam.kt b/core/src/main/kotlin/org/evomaster/core/problem/mcp/McpUriParam.kt index 46d9bbdcb7..da8df0a19a 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/mcp/McpUriParam.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/mcp/McpUriParam.kt @@ -3,6 +3,7 @@ package org.evomaster.core.problem.mcp import org.evomaster.core.problem.api.param.Param import org.evomaster.core.search.gene.Gene +/** Parameter representing a single URI template variable in a [McpResourceReadAction]. */ class McpUriParam(name: String, gene: Gene) : Param(name, gene) { override fun copyContent(): Param = McpUriParam(name, gene.copy()) } diff --git a/core/src/main/kotlin/org/evomaster/core/problem/mcp/client/HttpMcpClient.kt b/core/src/main/kotlin/org/evomaster/core/problem/mcp/client/HttpMcpClient.kt index f3b6a801a4..f6d77eafd1 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/mcp/client/HttpMcpClient.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/mcp/client/HttpMcpClient.kt @@ -1,28 +1,207 @@ package org.evomaster.core.problem.mcp.client -class HttpMcpClient(private val baseUrl: String) : McpClient { +import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.databind.ObjectMapper +import org.evomaster.core.problem.mcp.McpConst +import org.evomaster.core.remote.HttpClientFactory +import java.util.concurrent.atomic.AtomicInteger +import javax.ws.rs.client.Client +import javax.ws.rs.client.Entity +import javax.ws.rs.core.MediaType +import javax.ws.rs.core.Response +import kotlin.String +import kotlin.collections.List +import kotlin.collections.Map +/** + * [McpClient] implementation that uses the Streamable HTTP transport. + * + * All MCP messages are sent as HTTP POST requests to [baseUrl] using JSON-RPC 2.0, via the + * Jersey client from [HttpClientFactory]. + * + * **Session lifecycle**: [initialize] must be invoked once before any other method. It performs + * the two-step MCP handshake (`initialize` request + `notifications/initialized` notification) + * and captures the `Mcp-Session-Id` header returned by the server. + * + * @param baseUrl the full URL of the MCP endpoint. + * @param readTimeoutMs read timeout (in milliseconds) for the underlying Jersey client. + */ +class HttpMcpClient(private val baseUrl: String, readTimeoutMs: Int = 60_000) : McpClient { + + private val mapper: ObjectMapper = ObjectMapper() + private val idCounter = AtomicInteger(1) + + private val client: Client = HttpClientFactory.createTrustingJerseyClient(true, readTimeoutMs) + + @Volatile private var sessionId: String? = null + + private fun nextId() = idCounter.getAndIncrement() + + /** Send a JSON-RPC message. [id] is omitted for notifications. */ + private fun sendJsonRpc(method: String, params: Map, id: Int?, acceptEventStream: Boolean): Response { + val payload = mutableMapOf( + "jsonrpc" to McpConst.JSONRPC_VERSION, + "method" to method, + "params" to params + ) + id?.let { payload["id"] = it } + val body = mapper.writeValueAsString(payload) + + val acceptTypes = if (acceptEventStream) arrayOf(MediaType.APPLICATION_JSON, MediaType.SERVER_SENT_EVENTS) else arrayOf(MediaType.APPLICATION_JSON) + var builder = client.target(baseUrl).request(*acceptTypes) + sessionId?.let { builder = builder.header(McpConst.SESSION_ID_HEADER, it) } + + return builder.buildPost(Entity.entity(body, MediaType.APPLICATION_JSON_TYPE)).invoke() + } + + /** + * Perform the MCP initialization handshake (initialize + notifications/initialized). + * Must be called once before any other method. + */ fun initialize() { - throw UnsupportedOperationException("MCP server analysis is not yet supported") + val response = sendJsonRpc( + "initialize", + mapOf( + "protocolVersion" to McpConst.PROTOCOL_VERSION, + "capabilities" to emptyMap(), + "clientInfo" to mapOf("name" to "EvoMaster", "version" to "1.0.0") + ), + nextId(), + acceptEventStream = true + ) + val status = response.status + if (status >= 400) { + throw IllegalStateException( + "MCP initialize handshake failed with HTTP $status at '$baseUrl'" + ) + } + // Capture session ID before reading the body + response.getHeaderString(McpConst.SESSION_ID_HEADER)?.let { sessionId = it } + val responseBody = response.readEntity(String::class.java) + if (responseBody.isBlank()) { + throw IllegalStateException("MCP initialize handshake returned empty body") + } + // Send the required follow-up notification (fire-and-forget) + postNotification("notifications/initialized", emptyMap()) + } + + /** Send a JSON-RPC notification (no response expected). */ + private fun postNotification(method: String, params: Map) { + val response = sendJsonRpc(method, params, id = null, acceptEventStream = false) + // Discard the response to complete the HTTP exchange and release the connection + try { response.close() } catch (_: Exception) {} + } + + /** Send a JSON-RPC request and parse the response. */ + private fun post(method: String, params: Map = emptyMap()): Map? { + val response = sendJsonRpc(method, params, nextId(), acceptEventStream = true) + val responseBody = response.readEntity(String::class.java) + return mapper.readValue(responseBody, Map::class.java) as Map + } + + /** + * Fetches all pages from an MCP server response + * + * @param method the MCP method name + * @param resultKey the key in the result map containing the list of items + * @param transform function to convert each raw response to the target type + * @return list of all items + */ + private fun fetchPaginatedList( + method: String, + resultKey: String, + transform: (Map) -> T + ): List { + val results = mutableListOf() + var cursor: String? = null + + do { + // On first request cursor is null. + // On subsequent requests, include the cursor returned by the previous response. + val params = if (cursor != null) mapOf("cursor" to cursor) else emptyMap() + + val response = post(method, params) + ?: throw IllegalStateException("$method request failed: received null response") + + val result = response["result"] as? Map + ?: throw IllegalStateException("$method response missing 'result' field or invalid type") + + val items = result[resultKey] as? List> ?: emptyList() + items.forEach { item -> + results.add(transform(item)) + } + + // Extract cursor for next page + cursor = result["nextCursor"] as? String + } while (cursor != null) + + return results } override fun listTools(): List { - throw UnsupportedOperationException("MCP server analysis is not yet supported") + return fetchPaginatedList("tools/list", "tools", { tool -> + McpToolDefinition( + name = tool["name"] as String, + description = tool["description"] as String, + inputSchema = mapper.valueToTree(tool["inputSchema"]) + ) + }) } override fun listResources(): List { - throw UnsupportedOperationException("MCP server analysis is not yet supported") + return fetchPaginatedList("resources/list", "resources", { resource -> + McpResourceDefinition( + uri = resource["uri"] as String, + name = resource["name"] as String, + description = resource["description"] as? String ?: "", + mimeType = resource["mimeType"] as? String + ) + }) } override fun listResourceTemplates(): List { - throw UnsupportedOperationException("MCP server analysis is not yet supported") + return fetchPaginatedList("resources/templates/list", "resourceTemplates", { template -> + McpResourceTemplate( + uriTemplate = template["uriTemplate"] as String, + name = template["name"] as String, + description = template["description"] as? String ?: "" + ) + }) } override fun callTool(name: String, arguments: Map): McpToolResult { - throw UnsupportedOperationException("MCP server analysis is not yet supported") + val response = post("tools/call", mapOf("name" to name, "arguments" to arguments)) + ?: return McpToolResult(isError = true) + val result = response["result"] as? Map ?: return McpToolResult(isError = true) + val rawContent = result["content"] as? List> ?: emptyList() + val content = rawContent.map { c -> + val type = c["type"] as? String + ?: throw IllegalStateException("tools/call response content item missing required 'type' field") + McpContent( + type = type, + text = c["text"] as? String, + uri = c["uri"] as? String, + mimeType = c["mimeType"] as? String + ) + } + return McpToolResult( + content = content, + isError = result["isError"] as? Boolean ?: false + ) } override fun readResource(uri: String): McpResourceResult { - throw UnsupportedOperationException("MCP server analysis is not yet supported") + val response = post("resources/read", mapOf("uri" to uri)) + ?: return McpResourceResult() + val result = response["result"] as? Map ?: return McpResourceResult() + val rawContents = result["contents"] as? List> ?: emptyList() + val contents = rawContents.map { c -> + McpContent( + text = c["text"] as? String, + uri = c["uri"] as? String, + mimeType = c["mimeType"] as? String + ) + } + return McpResourceResult(contents = contents) } } diff --git a/core/src/main/kotlin/org/evomaster/core/problem/mcp/client/McpClient.kt b/core/src/main/kotlin/org/evomaster/core/problem/mcp/client/McpClient.kt index 8f0cc0cda7..365a2b67e3 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/mcp/client/McpClient.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/mcp/client/McpClient.kt @@ -1,5 +1,13 @@ package org.evomaster.core.problem.mcp.client +/** + * Abstraction over the MCP (Model Context Protocol) JSON-RPC transport. + * + * Defines the operations available for interaction with an MCP server: + * capability discovery ([listTools], [listResources], [listResourceTemplates]) and + * invocation ([callTool], [readResource]). + * + */ interface McpClient { fun listTools(): List fun listResources(): List diff --git a/core/src/main/kotlin/org/evomaster/core/problem/mcp/client/McpDTOs.kt b/core/src/main/kotlin/org/evomaster/core/problem/mcp/client/McpDTOs.kt index 5fea8e0f36..96a742f961 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/mcp/client/McpDTOs.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/mcp/client/McpDTOs.kt @@ -1,35 +1,43 @@ package org.evomaster.core.problem.mcp.client +import com.fasterxml.jackson.databind.JsonNode + +/** Tool definition as returned by the MCP `tools/list` response. */ data class McpToolDefinition( val name: String, - val description: String = "", - val inputSchema: Map = emptyMap() + val description: String, + val inputSchema: JsonNode ) +/** Static resource as returned by the MCP `resources/list` response. */ data class McpResourceDefinition( val uri: String, - val name: String = "", + val name: String, val description: String = "", val mimeType: String? = null ) +/** URI-template resource as returned by the MCP `resources/templates/list` response. */ data class McpResourceTemplate( val uriTemplate: String, - val name: String = "", + val name: String, val description: String = "" ) +/** Result of a `tools/call` invocation, as defined by the MCP specification. */ data class McpToolResult( val content: List = emptyList(), val isError: Boolean = false ) +/** Result of a `resources/read` invocation, as defined by the MCP specification. */ data class McpResourceResult( val contents: List = emptyList() ) +/** Content item within a tool or resource response. */ data class McpContent( - val type: String, + val type: String? = null, val text: String? = null, val uri: String? = null, val mimeType: String? = null diff --git a/core/src/main/kotlin/org/evomaster/core/problem/mcp/service/McpBlackBoxFitness.kt b/core/src/main/kotlin/org/evomaster/core/problem/mcp/service/McpBlackBoxFitness.kt index fe65b84d33..f236dc2d92 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/mcp/service/McpBlackBoxFitness.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/mcp/service/McpBlackBoxFitness.kt @@ -1,10 +1,38 @@ package org.evomaster.core.problem.mcp.service +import com.google.inject.Inject +import org.evomaster.core.problem.mcp.McpCallResult import org.evomaster.core.problem.mcp.McpIndividual +import org.evomaster.core.problem.mcp.McpResourceReadAction +import org.evomaster.core.problem.mcp.McpToolCallAction import org.evomaster.core.search.EvaluatedIndividual +import org.evomaster.core.search.FitnessValue +import org.evomaster.core.search.action.ActionResult +import org.evomaster.core.search.gene.BooleanGene +import org.evomaster.core.search.gene.Gene +import org.evomaster.core.search.gene.ObjectGene +import org.evomaster.core.search.gene.collection.ArrayGene +import org.evomaster.core.search.gene.numeric.NumberGene +import org.evomaster.core.search.gene.string.StringGene +import org.slf4j.Logger +import org.slf4j.LoggerFactory +/** + * Blackbox fitness function for MCP servers. + * + * Executes each action in an [McpIndividual] sequentially via the [McpSampler]'s client + * and scores coverage targets based on observable protocol signals (isError flag, + * successful reads). Mirrors [GraphQLBlackBoxFitness] in structure. + */ class McpBlackBoxFitness : McpFitness() { + companion object { + private val log: Logger = LoggerFactory.getLogger(McpBlackBoxFitness::class.java) + } + + @Inject + private lateinit var sampler: McpSampler + override fun doCalculateCoverage( individual: McpIndividual, targets: Set, @@ -12,6 +40,89 @@ class McpBlackBoxFitness : McpFitness() { fullyCovered: Boolean, descriptiveIds: Boolean, ): EvaluatedIndividual? { - throw UnsupportedOperationException("MCP server analysis is not yet supported") + + val fv = FitnessValue(individual.size().toDouble()) + val actionResults: MutableList = mutableListOf() + val client = sampler.getMcpClient() + + val actions = individual.seeMainExecutableActions() + + for (i in actions.indices) { + val action = actions[i] + val result = McpCallResult(action.getLocalId()) + actionResults.add(result) + + try { + when (action) { + is McpToolCallAction -> { + val args = geneToMap(action.inputSchema) + val toolResult = client.callTool(action.toolName, args) + + result.setIsError(toolResult.isError) + result.stopping = false + + val targetId = idMapper.handleLocalTarget("tool:${action.toolName}") + val score = if (!toolResult.isError) 1.0 else 0.5 + fv.updateTarget(targetId, score, i) + } + + is McpResourceReadAction -> { + val resourceResult = client.readResource(action.resolvedUri()) + result.setIsError(false) + result.stopping = false + + val targetId = idMapper.handleLocalTarget("resource:${action.id}") + // A successful read (even empty contents) scores 1.0 + fv.updateTarget(targetId, 1.0, i) + } + + else -> { + result.stopping = false + } + } + } catch (e: Exception) { + log.warn("Exception evaluating MCP action ${action.id}: ${e.message}") + result.setIsError(true) + result.stopping = true + break + } + } + + return EvaluatedIndividual( + fv, + individual.copy() as McpIndividual, + actionResults, + trackOperator = individual.trackOperator, + index = time.evaluatedIndividuals, + config = config + ) + } + + // ------------------------------------------------------------------------- + // Helpers + // ------------------------------------------------------------------------- + + /** + * Convert an [ObjectGene] to a [Map] of argument values suitable for passing + * to [McpClient.callTool]. Uses typed gene values when available, falling + * back to printable string representation for complex types. + */ + private fun geneToMap(gene: ObjectGene): Map { + val map = mutableMapOf() + for (field in gene.fields) { + map[field.name] = extractGeneValue(field) + } + return map + } + + private fun extractGeneValue(gene: Gene): Any? { + return when (gene) { + is StringGene -> gene.value + is BooleanGene -> gene.value + is NumberGene<*> -> gene.value + is ObjectGene -> geneToMap(gene) + is ArrayGene<*> -> gene.getViewOfElements().map { extractGeneValue(it) } + else -> gene.getValueAsPrintableString(listOf(), null, null, false) + } } } diff --git a/core/src/main/kotlin/org/evomaster/core/problem/mcp/service/McpFitness.kt b/core/src/main/kotlin/org/evomaster/core/problem/mcp/service/McpFitness.kt index dd09fa7764..ddb4e26ab0 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/mcp/service/McpFitness.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/mcp/service/McpFitness.kt @@ -3,4 +3,9 @@ package org.evomaster.core.problem.mcp.service import org.evomaster.core.problem.mcp.McpIndividual import org.evomaster.core.search.service.FitnessFunction +/** + * Abstract fitness function base for MCP testing. + * Concrete subclasses provide the actual coverage calculation strategy + * (blackbox, whitebox, etc.). + */ abstract class McpFitness : FitnessFunction() diff --git a/core/src/main/kotlin/org/evomaster/core/problem/mcp/service/McpSampler.kt b/core/src/main/kotlin/org/evomaster/core/problem/mcp/service/McpSampler.kt index e4b5e27c3a..1c0221e532 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/mcp/service/McpSampler.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/mcp/service/McpSampler.kt @@ -1,36 +1,220 @@ package org.evomaster.core.problem.mcp.service +import com.fasterxml.jackson.databind.JsonNode import org.evomaster.client.java.controller.api.dto.SutInfoDto import org.evomaster.core.problem.api.service.ApiWsSampler +import org.evomaster.core.remote.SutProblemException +import org.evomaster.core.problem.enterprise.EnterpriseActionGroup +import org.evomaster.core.problem.enterprise.SampleType +import org.evomaster.core.problem.mcp.McpAction import org.evomaster.core.problem.mcp.McpIndividual +import org.evomaster.core.problem.mcp.McpResourceReadAction +import org.evomaster.core.problem.mcp.McpToolCallAction +import org.evomaster.core.problem.mcp.McpUriParam import org.evomaster.core.problem.mcp.client.HttpMcpClient +import org.evomaster.core.search.action.ActionComponent +import org.evomaster.core.search.gene.Gene +import org.evomaster.core.search.gene.ObjectGene +import org.evomaster.core.search.gene.collection.ArrayGene +import org.evomaster.core.search.gene.numeric.IntegerGene +import org.evomaster.core.search.gene.BooleanGene +import org.evomaster.core.search.gene.string.StringGene +import org.slf4j.Logger +import org.slf4j.LoggerFactory import javax.annotation.PostConstruct +/** + * Sampler for MCP blackbox testing. + * + * On initialization ([initialize]), connects to the MCP server, performs the MCP handshake + * and discovers capabilities: + * - **Tools** (`tools/list`) → one [McpToolCallAction] per tool. + * - **Static resources** (`resources/list`) → one [McpResourceReadAction] per URI. + * - **Template resources** (`resources/templates/list`) → one [McpResourceReadAction] per template. + */ class McpSampler : ApiWsSampler() { + companion object { + private val log: Logger = LoggerFactory.getLogger(McpSampler::class.java) + } + private lateinit var mcpClient: HttpMcpClient + /** Actions for MCP tool calls, keyed by "tool:" */ + private val toolActionCluster: MutableMap = mutableMapOf() + + /** Actions for MCP resource reads, keyed by "resource:" or template key */ + private val resourceActionCluster: MutableMap = mutableMapOf() + + /** Pre-built single-call individuals, drained first in smartSample() */ + private val adHocInitialIndividuals: MutableList = mutableListOf() + @PostConstruct fun initialize() { - mcpClient = HttpMcpClient(config.base) - throw UnsupportedOperationException("MCP server analysis is not yet supported") + val name = McpSampler::class.simpleName + val url = config.base + log.debug("Initializing {}", name) + + + mcpClient = HttpMcpClient(url) + + actionCluster.clear() + toolActionCluster.clear() + resourceActionCluster.clear() + + // MCP requires initialize handshake before any other call + try { + mcpClient.initialize() + } catch (e: Exception) { + throw SutProblemException( + "Failed to initialize MCP session at '${config.base}'. Cause: ${e.message}" + ) + } + + // Discover tools + val tools = mcpClient.listTools() + for (tool in tools) { + val inputGene = buildObjectGeneFromSchema("input", tool.inputSchema) + val action = McpToolCallAction( + toolName = tool.name, + inputSchema = inputGene + ) + toolActionCluster[action.id] = action + actionCluster[action.id] = action + } + + // Discover static resources + val resources = mcpClient.listResources() + for (resource in resources) { + val action = McpResourceReadAction(uriTemplate = resource.uri, uriParams = emptyList(), isTemplate = false) + resourceActionCluster[action.id] = action + actionCluster[action.id] = action + } + + // Discover resource templates + val templates = mcpClient.listResourceTemplates() + for (template in templates) { + val paramNames = Regex("""\{(\w+)}""").findAll(template.uriTemplate).map { it.groupValues[1] }.toList() + val uriParams = paramNames.map { McpUriParam(it, StringGene(it)) } + val action = McpResourceReadAction(uriTemplate = template.uriTemplate, uriParams = uriParams, isTemplate = true) + // Use "template:" prefix to avoid collision with static resource keys + val key = "template:${template.uriTemplate}" + resourceActionCluster[key] = action + actionCluster[key] = action + } + + customizeAdHocInitialIndividuals() + + val toolQuantity = toolActionCluster.size + val resourceQuantity = resourceActionCluster.size + + log.debug("Done initializing {} — {} tools, {} resources", + name, toolQuantity, resourceQuantity) } + // ------------------------------------------------------------------------- + // Sampling + // ------------------------------------------------------------------------- + override fun sampleAtRandom(): McpIndividual { - throw UnsupportedOperationException("MCP server analysis is not yet supported") + val allActions: List = + toolActionCluster.values.toList() + resourceActionCluster.values.toList() + + if (allActions.isEmpty()) { + // Edge case: no capabilities discovered — return empty individual + val ind = McpIndividual(SampleType.RANDOM, mutableListOf()) + ind.doGlobalInitialize(searchGlobalState) + return ind + } + + val n = randomness.nextInt(1, getMaxTestSizeDuringSampler()) + val groups: MutableList = (0 until n).map { + val action = randomness.choose(allActions).copy() as McpAction + action.doInitialize(randomness) + makeGroup(action) + }.toMutableList() + + val ind = McpIndividual(SampleType.RANDOM, groups) + ind.doGlobalInitialize(searchGlobalState) + return ind } override fun smartSample(): McpIndividual { - throw UnsupportedOperationException("MCP server analysis is not yet supported") + if (adHocInitialIndividuals.isNotEmpty()) { + return adHocInitialIndividuals.removeAt(adHocInitialIndividuals.size - 1) + } + return sampleAtRandom() } - override fun hasSpecialInitForSmartSampler(): Boolean { - throw UnsupportedOperationException("MCP server analysis is not yet supported") - } + override fun hasSpecialInitForSmartSampler(): Boolean = adHocInitialIndividuals.isNotEmpty() override fun initSeededTests(infoDto: SutInfoDto?) { - throw UnsupportedOperationException("MCP server analysis is not yet supported") + throw UnsupportedOperationException("MCP seeded testing is not yet supported") + } + + // ------------------------------------------------------------------------- + // AdHoc individuals — one per discovered capability + // ------------------------------------------------------------------------- + + private fun customizeAdHocInitialIndividuals() { + adHocInitialIndividuals.clear() + + val mcpActions = toolActionCluster.values + resourceActionCluster.values + for (action in mcpActions) { + val copy = action.copy() as McpAction + copy.doInitialize(randomness) + val ind = McpIndividual(SampleType.RANDOM, mutableListOf(makeGroup(copy))) + ind.doGlobalInitialize(searchGlobalState) + adHocInitialIndividuals.add(ind) + } + } + + /** + * Wrap a single [McpAction] in an [EnterpriseActionGroup]. + * Uses the single-action convenience constructor. + */ + private fun makeGroup(action: McpAction): EnterpriseActionGroup { + return EnterpriseActionGroup(action) + } + + // ------------------------------------------------------------------------- + // Gene building from JSON Schema + // ------------------------------------------------------------------------- + + /** + * Recursively build a [Gene] from a JSON Schema node. + * + * Supported types: string, integer, number, boolean, object, array. + * Unknown or missing types fall back to [StringGene]. + */ + internal fun buildGeneFromSchema(name: String, schema: JsonNode): Gene { + val type = schema["type"]?.asText() + + return when (type) { + "string" -> StringGene(name) + "integer", "number" -> IntegerGene(name) + "boolean" -> BooleanGene(name) + "object" -> buildObjectGeneFromSchema(name, schema) + "array" -> ArrayGene(name, StringGene("element")) + else -> throw IllegalArgumentException("Unsupported type in tool $name input schema: $type") + } + } + + /** + * Build an [ObjectGene] from a JSON Schema node. + * If the schema has no "properties" key, returns an empty [ObjectGene]. + */ + internal fun buildObjectGeneFromSchema(name: String, schema: JsonNode): ObjectGene { + val properties = schema["properties"] ?: return ObjectGene(name, emptyList()) + val fields = properties.fields().asSequence().map { (propName, propSchema) -> + buildGeneFromSchema(propName, propSchema) + }.toList() + return ObjectGene(name, fields) } + // ------------------------------------------------------------------------- + // Expose client for fitness function + // ------------------------------------------------------------------------- + fun getMcpClient(): HttpMcpClient = mcpClient } diff --git a/core/src/test/kotlin/org/evomaster/core/problem/mcp/McpActionTest.kt b/core/src/test/kotlin/org/evomaster/core/problem/mcp/McpActionTest.kt new file mode 100644 index 0000000000..a4e488411a --- /dev/null +++ b/core/src/test/kotlin/org/evomaster/core/problem/mcp/McpActionTest.kt @@ -0,0 +1,55 @@ +package org.evomaster.core.problem.mcp + +import org.evomaster.core.search.gene.ObjectGene +import org.evomaster.core.search.gene.string.StringGene +import org.junit.jupiter.api.Assertions.* +import org.junit.jupiter.api.Test + +class McpActionTest { + + @Test + fun `McpToolCallAction getName returns tool colon toolName`() { + val inputGene = ObjectGene("input", emptyList()) + val action = McpToolCallAction("myTool", inputGene) + assertEquals("tool:myTool", action.getName()) + } + + @Test + fun `McpResourceReadAction getName starts with resource colon`() { + val action = McpResourceReadAction(uriTemplate = "http://example.com/resource", uriParams = emptyList()) + assertTrue(action.getName().startsWith("resource:")) + } + + @Test + fun `seeTopGenes on McpToolCallAction returns the ObjectGene`() { + val inputGene = ObjectGene("input", emptyList()) + val action = McpToolCallAction("myTool", inputGene) + val topGenes = action.seeTopGenes() + assertEquals(1, topGenes.size) + assertSame(inputGene, topGenes[0]) + } + + @Test + fun `copy on McpToolCallAction produces structurally equal but independent copy`() { + val inputGene = ObjectGene("input", emptyList()) + val action = McpToolCallAction("myTool", inputGene) + val copy = action.copy() as McpToolCallAction + + assertEquals(action.toolName, copy.toolName) + assertEquals(action.getName(), copy.getName()) + // Must be independent: different gene instances + assertNotSame(action.inputSchema, copy.inputSchema) + } + + @Test + fun `copy on McpResourceReadAction produces structurally equal but independent copy`() { + val param = McpUriParam("city", StringGene("city", "paris")) + val action = McpResourceReadAction(uriTemplate = "weather:///{city}/current", uriParams = listOf(param), isTemplate = true) + val copy = action.copy() as McpResourceReadAction + + assertEquals(action.isTemplate, copy.isTemplate) + assertEquals(action.uriTemplate, copy.uriTemplate) + assertNotSame(action.uriParams[0], copy.uriParams[0]) + assertEquals(action.resolvedUri(), copy.resolvedUri()) + } +} diff --git a/core/src/test/kotlin/org/evomaster/core/problem/mcp/McpIndividualTest.kt b/core/src/test/kotlin/org/evomaster/core/problem/mcp/McpIndividualTest.kt new file mode 100644 index 0000000000..8a25dd3d03 --- /dev/null +++ b/core/src/test/kotlin/org/evomaster/core/problem/mcp/McpIndividualTest.kt @@ -0,0 +1,76 @@ +package org.evomaster.core.problem.mcp + +import org.evomaster.core.problem.enterprise.EnterpriseActionGroup +import org.evomaster.core.problem.enterprise.SampleType +import org.evomaster.core.search.GroupsOfChildren +import org.evomaster.core.search.action.ActionComponent +import org.evomaster.core.search.gene.ObjectGene +import org.junit.jupiter.api.Assertions.* +import org.junit.jupiter.api.Test + +class McpIndividualTest { + + private fun buildIndividual(vararg actions: McpAction): McpIndividual { + val wrappedActions: MutableList> = + actions.map { EnterpriseActionGroup(it) }.toMutableList() + return McpIndividual( + sampleType = SampleType.RANDOM, + allActions = wrappedActions as MutableList + ) + } + + @Test + fun `seeMainExecutableActions returns all actions`() { + val toolAction = McpToolCallAction("myTool", ObjectGene("input", emptyList())) + val resourceAction = McpResourceReadAction(uriTemplate = "file:///data", uriParams = emptyList()) + + val individual = buildIndividual(toolAction, resourceAction) + + val mainActions = individual.seeMainExecutableActions() + assertEquals(2, mainActions.size) + } + + @Test + fun `copyContent returns equal-sized independent copy`() { + val toolAction = McpToolCallAction("myTool", ObjectGene("input", emptyList())) + val resourceAction = McpResourceReadAction(uriTemplate = "file:///data", uriParams = emptyList()) + + val individual = buildIndividual(toolAction, resourceAction) + val copy = individual.copy() as McpIndividual + + assertEquals( + individual.seeMainExecutableActions().size, + copy.seeMainExecutableActions().size + ) + // Must be independent: different action instances + assertNotSame( + individual.seeMainExecutableActions()[0], + copy.seeMainExecutableActions()[0] + ) + } + + @Test + fun `addMcpAction increases action count`() { + val toolAction = McpToolCallAction("myTool", ObjectGene("input", emptyList())) + val individual = buildIndividual(toolAction) + + assertEquals(1, individual.seeMainExecutableActions().size) + + val newAction = McpToolCallAction("anotherTool", ObjectGene("input2", emptyList())) + individual.addMcpAction(action = newAction) + + assertEquals(2, individual.seeMainExecutableActions().size) + } + + @Test + fun `group size is tracked correctly`() { + val toolAction = McpToolCallAction("myTool", ObjectGene("input", emptyList())) + val resourceAction = McpResourceReadAction(uriTemplate = "file:///data", uriParams = emptyList()) + + val individual = buildIndividual(toolAction, resourceAction) + + assertEquals(2, individual.groupsView()!!.sizeOfGroup(GroupsOfChildren.MAIN)) + assertEquals(0, individual.groupsView()!!.sizeOfGroup(GroupsOfChildren.INITIALIZATION_SQL)) + assertEquals(0, individual.groupsView()!!.sizeOfGroup(GroupsOfChildren.INITIALIZATION_MONGO)) + } +} diff --git a/core/src/test/kotlin/org/evomaster/core/problem/mcp/client/HttpMcpClientTest.kt b/core/src/test/kotlin/org/evomaster/core/problem/mcp/client/HttpMcpClientTest.kt new file mode 100644 index 0000000000..b03e7eafd3 --- /dev/null +++ b/core/src/test/kotlin/org/evomaster/core/problem/mcp/client/HttpMcpClientTest.kt @@ -0,0 +1,183 @@ +package org.evomaster.core.problem.mcp.client + +import com.github.tomakehurst.wiremock.WireMockServer +import com.github.tomakehurst.wiremock.client.WireMock +import com.github.tomakehurst.wiremock.client.WireMock.* +import com.github.tomakehurst.wiremock.core.WireMockConfiguration +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.Assertions.* +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test + +class HttpMcpClientTest { + + private lateinit var wm: WireMockServer + private lateinit var client: HttpMcpClient + + @BeforeEach + fun setUp() { + wm = WireMockServer(WireMockConfiguration().dynamicPort()) + wm.start() + client = HttpMcpClient("http://localhost:${wm.port()}/mcp") + } + + @AfterEach + fun tearDown() { + wm.stop() + } + + private fun stubPost(responseBody: String) { + wm.stubFor( + WireMock.post(urlEqualTo("/mcp")) + .willReturn( + aResponse() + .withStatus(200) + .withHeader("Content-Type", "application/json") + .withBody(responseBody) + ) + ) + } + + @Test + fun testListToolsParsesDefinitionsCorrectly() { + stubPost( + """{"jsonrpc":"2.0","result":{"tools":[{"name":"foo","description":"bar","inputSchema":{}}]},"id":1}""" + ) + + val tools = client.listTools() + + assertEquals(1, tools.size) + assertEquals("foo", tools[0].name) + assertEquals("bar", tools[0].description) + } + + @Test + fun testListToolsHandlesPagination() { + // First page + wm.stubFor( + WireMock.post(urlEqualTo("/mcp")) + .inScenario("pagination") + .whenScenarioStateIs(com.github.tomakehurst.wiremock.stubbing.Scenario.STARTED) + .willReturn( + aResponse() + .withStatus(200) + .withHeader("Content-Type", "application/json") + .withBody( + """{"jsonrpc":"2.0","result":{"tools":[{"name":"tool1","description":"","inputSchema":{}}],"nextCursor":"page2"},"id":1}""" + ) + ) + .willSetStateTo("page2") + ) + // Second page + wm.stubFor( + WireMock.post(urlEqualTo("/mcp")) + .inScenario("pagination") + .whenScenarioStateIs("page2") + .willReturn( + aResponse() + .withStatus(200) + .withHeader("Content-Type", "application/json") + .withBody( + """{"jsonrpc":"2.0","result":{"tools":[{"name":"tool2","description":"","inputSchema":{}}]},"id":2}""" + ) + ) + ) + + val tools = client.listTools() + + assertEquals(2, tools.size) + assertEquals("tool1", tools[0].name) + assertEquals("tool2", tools[1].name) + } + + @Test + fun testCallToolReturnsSuccessResult() { + stubPost( + """{"jsonrpc":"2.0","result":{"content":[{"type":"text","text":"hello"}],"isError":false},"id":1}""" + ) + + val result = client.callTool("foo", mapOf("arg" to "value")) + + assertFalse(result.isError) + assertEquals(1, result.content.size) + assertEquals("text", result.content[0].type) + assertEquals("hello", result.content[0].text) + } + + @Test + fun testCallToolReturnsErrorResult() { + stubPost( + """{"jsonrpc":"2.0","result":{"content":[{"type":"text","text":"error message"}],"isError":true},"id":1}""" + ) + + val result = client.callTool("foo", emptyMap()) + + assertTrue(result.isError) + assertEquals(1, result.content.size) + assertEquals("error message", result.content[0].text) + } + + @Test + fun testCallToolWithMissingResultReturnsError() { + stubPost( + """{"jsonrpc":"2.0","error":{"code":-32601,"message":"Method not found"},"id":1}""" + ) + + val result = client.callTool("nonexistent", emptyMap()) + + assertTrue(result.isError) + assertTrue(result.content.isEmpty()) + } + + @Test + fun testReadResourceParsesContentCorrectly() { + stubPost( + """{"jsonrpc":"2.0","result":{"contents":[{"text":"resource content","uri":"file:///data/res","mimeType":"text/plain"}]},"id":1}""" + ) + + val result = client.readResource("file:///data/res") + + assertEquals(1, result.contents.size) + assertEquals("resource content", result.contents[0].text) + assertEquals("file:///data/res", result.contents[0].uri) + assertEquals("text/plain", result.contents[0].mimeType) + } + + @Test + fun testReadResourceWithMissingResultReturnsEmpty() { + stubPost( + """{"jsonrpc":"2.0","error":{"code":-32602,"message":"Invalid params"},"id":1}""" + ) + + val result = client.readResource("unknown://uri") + + assertTrue(result.contents.isEmpty()) + } + + @Test + fun testListResourcesParsesDefinitionsCorrectly() { + stubPost( + """{"jsonrpc":"2.0","result":{"resources":[{"uri":"file:///data","name":"data","description":"A data resource","mimeType":"application/json"}]},"id":1}""" + ) + + val resources = client.listResources() + + assertEquals(1, resources.size) + assertEquals("file:///data", resources[0].uri) + assertEquals("data", resources[0].name) + assertEquals("application/json", resources[0].mimeType) + } + + @Test + fun testListResourceTemplatesParsesTemplatesCorrectly() { + stubPost( + """{"jsonrpc":"2.0","result":{"resourceTemplates":[{"uriTemplate":"file:///{path}","name":"fileTemplate","description":"A file template"}]},"id":1}""" + ) + + val templates = client.listResourceTemplates() + + assertEquals(1, templates.size) + assertEquals("file:///{path}", templates[0].uriTemplate) + assertEquals("fileTemplate", templates[0].name) + } +}