From 908964c61715912933e48c501955364891991ee3 Mon Sep 17 00:00:00 2001 From: HarshPopat23 Date: Sun, 25 Oct 2026 09:44:40 +0530 Subject: [PATCH 1/2] Benchmark MCP protocol helpers Add gated MCP benchmarks for parsed metadata, separate JSON parsing, header mirroring, result writers and subscription intersection. Include Release-active fixture controls and reproducible measurement instructions. Signed-off-by: HarshPopat23 --- benchmark/CMakeLists.txt | 9 ++ benchmark/MCP.markdown | 76 ++++++++++++ benchmark/mcp.cc | 252 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 337 insertions(+) create mode 100644 benchmark/MCP.markdown create mode 100644 benchmark/mcp.cc diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt index b4aedefec..655cfe1d0 100644 --- a/benchmark/CMakeLists.txt +++ b/benchmark/CMakeLists.txt @@ -1,5 +1,9 @@ set(BENCHMARK_SOURCES) +if(SOURCEMETA_CORE_MCP) + list(APPEND BENCHMARK_SOURCES mcp.cc) +endif() + if(SOURCEMETA_CORE_REGEX) list(APPEND BENCHMARK_SOURCES regex.cc) endif() @@ -66,6 +70,11 @@ if(BENCHMARK_SOURCES) target_compile_definitions(sourcemeta_core_benchmarks PRIVATE CURRENT_DIRECTORY="${CMAKE_CURRENT_SOURCE_DIR}") + if(SOURCEMETA_CORE_MCP) + target_link_libraries(sourcemeta_core_benchmarks + PRIVATE sourcemeta::core::mcp) + endif() + if(SOURCEMETA_CORE_REGEX) target_link_libraries(sourcemeta_core_benchmarks PRIVATE sourcemeta::core::regex) diff --git a/benchmark/MCP.markdown b/benchmark/MCP.markdown new file mode 100644 index 000000000..f07df6404 --- /dev/null +++ b/benchmark/MCP.markdown @@ -0,0 +1,76 @@ +# MCP performance baselines + +The `MCP_` cases use the existing Core benchmark harness. They measure local +protocol helpers, not model generation, networking, a transport/session runtime, +or application JSON Schema evaluation. No production code is optimized here. +Fixtures are deterministic, authored microbenchmark inputs, not captured traffic. + +## Run and record + +Configure a Release build with MCP and benchmarks enabled: + +```sh +cmake -S . -B build -DCMAKE_BUILD_TYPE=Release \ + -DSOURCEMETA_CORE_MCP=ON -DSOURCEMETA_CORE_BENCHMARK=ON \ + -DSOURCEMETA_CORE_USE_SYSTEM_ALLOCATOR=ON +cmake --build build --config Release --target sourcemeta_core_benchmarks +# Single-configuration generators; multi-configuration builds may add Release/ +./build/benchmark/sourcemeta_core_benchmarks --filter MCP_ --output build/mcp.json +``` + +Keep the JSON output alongside the commit SHA, complete configure command, +`cmake -LA -N build` output, compiler/version (`c++ --version` or the platform +compiler equivalent), CPU/OS, build configuration, SIMD flags, and system load. +The command above explicitly selects the system allocator. Repeat in a separate +build with `SOURCEMETA_CORE_USE_SYSTEM_ALLOCATOR=OFF` to study the configured +mimalloc allocator; do not mix allocator/build/compiler results. The harness +JSON records timings and core count, not all of this environment information. + +Run the filtered executable multiple times on an otherwise idle machine. The +harness reports per-operation real/CPU time, a cold measurement, and the fastest +of three steady-state measurement windows; it does **not** report request p95 or +allocation counts. Use an external allocation profiler separately if needed, +and do not compare profiler timings with uninstrumented runs. There are no new +noisy timing thresholds in CI. MCP-disabled builds do not register these cases. + +## Workloads and measurement boundaries + +All fixture construction and preflight correctness checks occur before the +state loop. Checks throw on failure even with `NDEBUG`. Results are consumed +through the harness optimization barrier. Per-iteration output destruction is +included. Counts below are part of the benchmark names. + +| Cases | Small / large | Timed operation | +|---|---|---| +| `MCP_Parse_Metadata_*` | 0 / 128 extra metadata keys | Parse cached UTF-8 JSON and destroy the DOM; no MCP validation | +| `MCP_Validate_Metadata_*` | 0 / 128 extra keys | Validate an already parsed request, including metadata-key checks | +| `MCP_Reject_Metadata_*` | 0 / 128 extra keys | Reject object-valued `progressToken`, after other metadata checks | +| `MCP_Make_Headers_*` | 2 / 64 descriptors | Descriptor checks, header creation and plain/Unicode sentinel encoding | +| `MCP_Validate_Headers_*` | 2 / 64 descriptors | Validate already parsed arguments against 4 / 128 borrowed headers | +| `MCP_Reject_Headers_*` | 2 / 64 descriptors | Same, with a mismatch on the last mirrored parameter; includes error construction | +| `MCP_Write_Result_*` | 1 / 256 resources | Const-input result validation, copying, decoration, JSON-RPC envelope and ostream serialization | +| `MCP_Copy_Move_Write_Result_*` | 1 / 256 resources | Explicit fresh-payload copy plus move-taking writer; not a copy-free move comparison | +| `MCP_Reject_Result_*` | 1 / 256 resources | Invalid URI type on the last entry; includes exception construction/catching, not successful serialization | +| `MCP_Intersect_Overlap_*` | 8 / 512 URIs per side | Intersection with half overlap, preserving requested ordering | +| `MCP_Intersect_Disjoint_*` | 8 / 512 per side | Empty intersection; a valid no-match result, not malformed input | + +Each serialization iteration creates and destroys an `ostringstream`; its buffer +allocation is included. The successful const path is round-trip checked before +timing. Both writer variants receive the same source payload. The move-taking +path explicitly includes the fresh input copy required for repeated iterations. +These are whole public-API operations, not isolated JSON stringify kernels. + +The valid metadata wire sizes are **185 / 10,187 bytes** (compact UTF-8 JSON). +Each extra key is `bench-{index}` with a 64-byte ASCII value. Rejection adds +`progressToken:{}`. Resource payloads before envelope/cache decoration are +**201 / 48,435 bytes**; each resource has a name, a `file:///bench/{index}` URI +and a 128-byte description. Headers mirror alternating `plain` and ` café ` +values; half require UTF-8/base64 sentinel encoding. An equal number of unrelated +headers precedes mirrored headers to exercise lookup work. These sizes exclude +HTTP framing and are not protocol payload limits. + +Subscription URIs are `file:///bench/{index}`. Requested IDs are `[0,N)`; +supported IDs are `[N/2,3N/2)` for overlap and `[N,2N)` for disjoint workloads. +Owners are fully populated before any borrowed views are constructed. Reuse +these exact fixture shapes for before/after comparisons; do not attribute gains +from changing payloads, validation policy or included setup to an optimization. diff --git a/benchmark/mcp.cc b/benchmark/mcp.cc new file mode 100644 index 000000000..06ce10f68 --- /dev/null +++ b/benchmark/mcp.cc @@ -0,0 +1,252 @@ +#include +#include +#include + +#include // std::size_t +#include // std::ostringstream +#include // std::runtime_error +#include // std::string, std::to_string +#include // std::pair, std::move +#include // std::vector + +namespace { +using namespace sourcemeta::core; +constexpr auto VERSION{MCPProtocolVersion::V_2026_07_28}; + +// Unlike assert, preflight controls must also run in Release builds. All setup +// and controls precede the state loop, so they are outside its timing window. +auto check(const bool condition) -> void { + if (!condition) { + throw std::runtime_error{"Invalid MCP benchmark fixture"}; + } +} + +auto metadata(const std::size_t count, const bool invalid) -> JSON { + auto value{parse_json(R"JSON({"jsonrpc":"2.0","id":1,"method":"tools/call", + "params":{"_meta":{"io.modelcontextprotocol/protocolVersion":"2026-07-28", + "io.modelcontextprotocol/clientCapabilities":{}},"name":"bench"}})JSON")}; + auto &meta{value.at("params").at("_meta")}; + for (std::size_t index{0}; index < count; ++index) { + meta.assign("bench-" + std::to_string(index), JSON{std::string(64, 'x')}); + } + if (invalid) { + meta.assign("progressToken", JSON::make_object()); + } + return value; +} + +auto validate_metadata(BenchmarkState &state, const std::size_t count, + const bool invalid) -> void { + const auto input{metadata(count, invalid)}; + const auto expected{invalid ? MCPRequestMetaStatus::InvalidProgressToken + : MCPRequestMetaStatus::Valid}; + check(mcp_validate_request_meta(input).first == expected); + for (auto iteration : state) { + const auto result{mcp_validate_request_meta(input)}; + benchmark_do_not_optimize(result); + } +} + +auto parse_metadata(BenchmarkState &state, const std::size_t count) -> void { + const auto input{metadata(count, false)}; + std::ostringstream encoded; + stringify(input, encoded); + const auto wire{encoded.str()}; + check(wire.size() == (count == 0 ? 185 : 10187)); + check(parse_json(wire) == input); + for (auto iteration : state) { + const auto result{parse_json(wire)}; + benchmark_do_not_optimize(result); + } +} + +// Owning strings are fully populated before constructing views. No owner is +// mutated after its views have been handed to descriptors/headers. +struct HeaderFixture { + std::vector names; + std::vector descriptors; + JSON arguments{JSON::make_object()}; + std::vector> owned; + std::vector> headers; + + explicit HeaderFixture(const std::size_t count, const bool invalid) { + for (std::size_t index{0}; index < count; ++index) { + names.push_back("field" + std::to_string(index)); + arguments.assign(names.back(), JSON{index % 2 == 0 ? "plain" : " café "}); + } + for (const auto &name : names) { + descriptors.push_back({name, {name}, JSON::Type::String}); + } + // Unrelated headers force the validator to search, not just match the + // first element. The invalid case fails at the last mirrored parameter. + for (std::size_t index{0}; index < count; ++index) { + owned.emplace_back("Other-" + std::to_string(index), "ignored"); + } + auto mirrored{mcp_make_parameter_headers(descriptors, arguments)}; + for (auto &entry : mirrored) { + owned.push_back(std::move(entry)); + } + if (invalid) { + owned.back().second = "mismatch"; + } + for (const auto &entry : owned) { + headers.emplace_back(entry.first, entry.second); + } + } +}; + +auto validate_headers(BenchmarkState &state, const std::size_t count, + const bool invalid) -> void { + const HeaderFixture input{count, invalid}; + const JSON identifier{1}; + check(mcp_validate_parameter_headers(VERSION, identifier, input.descriptors, + input.arguments, input.headers) + .has_value() == invalid); + for (auto iteration : state) { + const auto result{ + mcp_validate_parameter_headers(VERSION, identifier, input.descriptors, + input.arguments, input.headers)}; + benchmark_do_not_optimize(result); + } +} + +auto make_headers(BenchmarkState &state, const std::size_t count) -> void { + const HeaderFixture input{count, false}; + check(mcp_make_parameter_headers(input.descriptors, input.arguments).size() == + count); + for (auto iteration : state) { + const auto result{ + mcp_make_parameter_headers(input.descriptors, input.arguments)}; + benchmark_do_not_optimize(result); + } +} + +auto result_payload(const std::size_t count) -> JSON { + auto entries{JSON::make_array()}; + for (std::size_t index{0}; index < count; ++index) { + auto entry{JSON::make_object()}; + entry.assign("name", JSON{"item" + std::to_string(index)}); + entry.assign("uri", JSON{"file:///bench/" + std::to_string(index)}); + entry.assign("description", JSON{std::string(128, 'x')}); + entries.push_back(std::move(entry)); + } + auto result{JSON::make_object()}; + result.assign("resources", std::move(entries)); + return result; +} + +auto serialize_result(BenchmarkState &state, const std::size_t count, + const bool move, const bool invalid) -> void { + auto input{result_payload(count)}; + if (invalid) { + input.at("resources").at(count - 1).assign("uri", JSON{false}); + } + const JSON identifier{1}; + const MCPCachePolicy policy{.ttl_ms = 0, .scope = MCPCacheScope::Private}; + std::ostringstream control; + bool rejected{false}; + try { + mcp_write_result(control, VERSION, "resources/list", identifier, input, + policy); + } catch (const MCPError &) { + rejected = true; + } + check(rejected == invalid); + if (!invalid) { + std::ostringstream payload; + stringify(input, payload); + check(payload.str().size() == (count == 1 ? 201 : 48435)); + const auto output{parse_json(control.str())}; + check(output.at("result").at("resources") == input.at("resources")); + if (move) { + auto fresh{input}; + std::ostringstream moved; + mcp_write_result(moved, VERSION, "resources/list", identifier, + std::move(fresh), policy); + check(moved.str() == control.str()); + } + } + for (auto iteration : state) { + std::ostringstream stream; + if (invalid) { + try { + mcp_write_result(stream, VERSION, "resources/list", identifier, input, + policy); + } catch (const MCPError &error) { + benchmark_do_not_optimize(error); + } + } else if (move) { + // Each repetition needs an intact payload. Include this copy explicitly; + // this is copy + move-write, not a fictitious copy-free move comparison. + auto fresh{input}; + mcp_write_result(stream, VERSION, "resources/list", identifier, + std::move(fresh), policy); + } else { + mcp_write_result(stream, VERSION, "resources/list", identifier, input, + policy); + } + benchmark_do_not_optimize(stream); + } +} + +auto intersect(BenchmarkState &state, const std::size_t count, + const bool overlap) -> void { + std::vector requested; + std::vector supported; + for (std::size_t index{0}; index < count; ++index) { + requested.push_back("file:///bench/" + std::to_string(index)); + supported.push_back("file:///bench/" + + std::to_string(index + (overlap ? count / 2 : count))); + } + MCPSubscriptionFilter left; + MCPSubscriptionFilter right; + left.resource_subscriptions.emplace(); + right.resource_subscriptions.emplace(); + for (const auto &uri : requested) { + left.resource_subscriptions->push_back(uri); + } + for (const auto &uri : supported) { + right.resource_subscriptions->push_back(uri); + } + const auto control{mcp_intersect_subscription_filters(left, right)}; + check(control.resource_subscriptions && + control.resource_subscriptions->size() == (overlap ? count / 2 : 0)); + if (overlap) { + for (std::size_t index{0}; index < count / 2; ++index) { + check(control.resource_subscriptions->at(index) == + requested.at(index + count / 2)); + } + } + for (auto iteration : state) { + const auto result{mcp_intersect_subscription_filters(left, right)}; + benchmark_do_not_optimize(result); + } +} +} // namespace + +BENCHMARK(MCP_Parse_Metadata_0) { parse_metadata(state, 0); } +BENCHMARK(MCP_Parse_Metadata_128) { parse_metadata(state, 128); } +BENCHMARK(MCP_Validate_Metadata_0) { validate_metadata(state, 0, false); } +BENCHMARK(MCP_Validate_Metadata_128) { validate_metadata(state, 128, false); } +BENCHMARK(MCP_Reject_Metadata_0) { validate_metadata(state, 0, true); } +BENCHMARK(MCP_Reject_Metadata_128) { validate_metadata(state, 128, true); } +BENCHMARK(MCP_Validate_Headers_2) { validate_headers(state, 2, false); } +BENCHMARK(MCP_Validate_Headers_64) { validate_headers(state, 64, false); } +BENCHMARK(MCP_Reject_Headers_2) { validate_headers(state, 2, true); } +BENCHMARK(MCP_Reject_Headers_64) { validate_headers(state, 64, true); } +BENCHMARK(MCP_Make_Headers_2) { make_headers(state, 2); } +BENCHMARK(MCP_Make_Headers_64) { make_headers(state, 64); } +BENCHMARK(MCP_Write_Result_1) { serialize_result(state, 1, false, false); } +BENCHMARK(MCP_Write_Result_256) { serialize_result(state, 256, false, false); } +BENCHMARK(MCP_Copy_Move_Write_Result_1) { + serialize_result(state, 1, true, false); +} +BENCHMARK(MCP_Copy_Move_Write_Result_256) { + serialize_result(state, 256, true, false); +} +BENCHMARK(MCP_Reject_Result_1) { serialize_result(state, 1, false, true); } +BENCHMARK(MCP_Reject_Result_256) { serialize_result(state, 256, false, true); } +BENCHMARK(MCP_Intersect_Overlap_8) { intersect(state, 8, true); } +BENCHMARK(MCP_Intersect_Overlap_512) { intersect(state, 512, true); } +BENCHMARK(MCP_Intersect_Disjoint_8) { intersect(state, 8, false); } +BENCHMARK(MCP_Intersect_Disjoint_512) { intersect(state, 512, false); } From d55f3ed65395e69ed5dd35bce83971b001e0bf6c Mon Sep 17 00:00:00 2001 From: HarshPopat23 Date: Sun, 25 Oct 2026 10:06:42 +0530 Subject: [PATCH 2/2] Fix MCP benchmark clang-tidy diagnostics Use a designated parameter descriptor initializer and make arithmetic precedence explicit for the macOS LLVM warnings-as-errors build. Signed-off-by: HarshPopat23 --- benchmark/mcp.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/benchmark/mcp.cc b/benchmark/mcp.cc index 06ce10f68..b2064ca4c 100644 --- a/benchmark/mcp.cc +++ b/benchmark/mcp.cc @@ -75,7 +75,8 @@ struct HeaderFixture { arguments.assign(names.back(), JSON{index % 2 == 0 ? "plain" : " café "}); } for (const auto &name : names) { - descriptors.push_back({name, {name}, JSON::Type::String}); + descriptors.push_back( + {.name = name, .path = {name}, .type = JSON::Type::String}); } // Unrelated headers force the validator to search, not just match the // first element. The invalid case fails at the last mirrored parameter. @@ -214,7 +215,7 @@ auto intersect(BenchmarkState &state, const std::size_t count, if (overlap) { for (std::size_t index{0}; index < count / 2; ++index) { check(control.resource_subscriptions->at(index) == - requested.at(index + count / 2)); + requested.at(index + (count / 2))); } } for (auto iteration : state) {