Skip to content

Commit 6c3c272

Browse files
committed
AUS-1012: JSON results from every benchmark mode, the publish profile, ingest and generated README tables
1 parent 5856bf9 commit 6c3c272

44 files changed

Lines changed: 2933 additions & 207 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎Json-Rpc/README.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -140,13 +140,16 @@ The 2.0 runs used the built-in serializer on an AMD Ryzen 7 7800X3D with .NET 10
140140

141141
![JSON-RPC.Net 1.2.3 and 2.0 throughput through the string and byte entry points](https://raw.githubusercontent.com/Astn/JSON-RPC.NET/master/benchmarks/charts/headline-1x-vs-2.svg)
142142

143+
<!-- benchmarks:headline -->
143144
| Path | RPC/s | Against 1.2.3 |
144145
| --- | ---: | ---: |
145146
| 1.2.3, `Task<string> Process(string)`, thread pool, best batch size | 3.08 M | |
146147
| 2.0, the same string API and the same loop | 13.3 M | 4.3× |
147148
| 2.0, `Process(bytes)`, 16 dedicated threads | 31.7 M | 10.3× |
148149
| 2.0, `ProcessAsync(bytes)`, 16 awaited workers | 32.1 M | 10.4× |
149150

151+
<!-- /benchmarks:headline -->
152+
150153
These measurements cover the library without a transport.
151154
The asynchronous byte row uses methods that complete inline.
152155
For Kestrel TCP with 256 requests in flight per connection and `EnableAsyncMethods = false`,

‎README.md‎

Lines changed: 28 additions & 4 deletions
Large diffs are not rendered by default.

‎TestServer_Console/AsyncBenchmark.cs‎

Lines changed: 67 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
using System.Diagnostics;
44
using System.Linq;
55
using System.Text;
6+
using System.Text.Json.Nodes;
67
using System.Threading;
78
using System.Threading.Tasks;
89
using AustinHarris.JsonRpc;
@@ -47,33 +48,53 @@ internal static async Task RunAsync(Action<string> print, double seconds, int wo
4748
{
4849
if (seconds <= 0 || workers <= 0) throw new ArgumentOutOfRangeException(nameof(seconds));
4950
var summary = new List<BenchmarkRunner.ChartRow>();
51+
var resultRows = new JsonArray();
5052
foreach (var (shape, flow) in Rows)
5153
{
5254
string session = "async-benchmark-" + shape + "-" + flow;
5355
try
5456
{
5557
Register(session, shape, flow);
5658
var rowInputs = Inputs(shape);
57-
await ValidateAndReportAllocations(print, session, shape, flow, rowInputs);
59+
var perShape = await ValidateAndReportAllocations(print, session, shape, flow, rowInputs);
5860
bool yield = shape == "yield";
5961
var m = await Measure(session, rowInputs, workers, seconds, processWide: yield);
6062
string label = yield ? "one request (yieldsOnce)" : "equal-weight mix of five requests";
6163
print($"{shape} / {flow}: {m.Count:N0} RPCs, {workers} workers, {m.Seconds:F3} s, {m.RpcPerSec:N0} RPC/s; worker startup {m.StartupSeconds * 1000:F1} ms; " +
6264
$"{(yield ? "process-wide" : "worker-thread")} allocation total {m.AllocatedBytes:N0} B ({m.BytesPerRpc:F0} B/RPC, including method and harness allocations); {label}; {m.Cpu.TrailingText}.");
6365
summary.Add(new BenchmarkRunner.ChartRow($"{shape} / {flow}", m.RpcPerSec, $"{m.Count,12:N0} RPCs {m.BytesPerRpc,6:F0} B/RPC {m.Cpu.TrailingText}"));
66+
var resultRow = BenchResults.Row(RowId(workers == 1 ? "async1" : "asyncn", shape, flow), m.RpcPerSec, m.Count, m.Cpu);
67+
resultRow.Insert(1, "shape", shape);
68+
resultRow.Insert(2, "flow", flow.ToString());
69+
resultRow.Insert(5, "bytesPerRpc", Math.Round(m.BytesPerRpc));
70+
resultRow.Insert(6, "startupMs", Math.Round(m.StartupSeconds * 1000, 1));
71+
if (perShape != null) resultRow["bytesPerShape"] = BenchResults.Numbers(perShape);
72+
resultRows.Add(resultRow);
6473
}
6574
finally { Handler.DestroySession(session); }
6675
}
6776
BenchmarkRunner.PrintBarChart($"ProcessAsync(bytes), awaited workers - {Config.Serializer.Name} - {workers} worker{(workers == 1 ? "" : "s")} - RPC/s by registration", "Registration", summary);
77+
78+
if (BenchResults.Enabled)
79+
{
80+
var doc = BenchResults.Document("async", seconds);
81+
doc["workers"] = workers;
82+
doc["rows"] = resultRows;
83+
BenchResults.Write($"async-w{workers}", doc, print);
84+
}
6885
}
6986

87+
/// <summary>The results id of a registration row: <c>&lt;prefix&gt;-&lt;shape&gt;-&lt;flow&gt;</c> in lower case, as benchmarks.json names the series.</summary>
88+
internal static string RowId(string prefix, string shape, RpcContextFlow flow) => $"{prefix}-{shape.ToLowerInvariant()}-{flow.ToString().ToLowerInvariant()}";
89+
7090
/// <summary>
7191
/// The scaling gate: the inline None rows at 1, 2 and <paramref name="workers"/> workers, <paramref name="runs"/>
7292
/// paired runs, medians, and the 2/1 and N/1 ratios. Returns false when any N/1 ratio is below
7393
/// <paramref name="threshold"/>. A process-wide serialization point on the async path holds the ratio near 1.3
74-
/// on any core count; the per-thread cache measured about 6.8 on 16 threads.
94+
/// on any core count; the per-thread cache measured about 6.8 on 16 threads. When <paramref name="report"/> is given,
95+
/// the gate cells (median, the paired runs, CPU), the ratios and the verdict are added to it for the results file.
7596
/// </summary>
76-
internal static async Task<bool> ScaleAsync(Action<string> print, double seconds, int workers, double threshold, int runs = 3)
97+
internal static async Task<bool> ScaleAsync(Action<string> print, double seconds, int workers, double threshold, int runs = 3, JsonObject report = null)
7798
{
7899
if (workers < 2) throw new ArgumentOutOfRangeException(nameof(workers));
79100
var counts = new[] { 1, 2, workers }.Distinct().ToArray();
@@ -101,6 +122,8 @@ internal static async Task<bool> ScaleAsync(Action<string> print, double seconds
101122
}
102123

103124
bool pass = true;
125+
var cells = new JsonArray();
126+
var gate = new JsonArray();
104127
print("");
105128
print($"| Registration | 1 worker | 2 workers | {workers} workers | 2/1 | {workers}/1 | gate | CPU at {workers} workers |");
106129
print("| --- | ---: | ---: | ---: | ---: | ---: | --- | --- |");
@@ -113,11 +136,32 @@ internal static async Task<bool> ScaleAsync(Action<string> print, double seconds
113136
bool ok = many / one >= threshold;
114137
pass &= ok;
115138
print($"| {shape} / {flow} | {one:N0} | {two:N0} | {many:N0} | {two / one:F2} | {many / one:F2} | {(ok ? "pass" : "FAIL")} | {cpu.TrailingText} |");
139+
foreach (int w in counts)
140+
{
141+
var list = results[(shape, flow, w)];
142+
var cell = BenchResults.Row(RowId("scale", shape, flow), Median(list.Select(m => m.RpcPerSec).ToList()), list.Sum(m => m.Count), MedianCpu(list));
143+
cell.Insert(1, "shape", shape);
144+
cell.Insert(2, "flow", flow.ToString());
145+
cell.Insert(3, "workers", w);
146+
cell.Insert(5, "values", BenchResults.Numbers(list.Select(m => Math.Round(m.RpcPerSec))));
147+
cells.Add(cell);
148+
}
149+
gate.Add(new JsonObject { ["shape"] = shape, ["flow"] = flow.ToString(), ["twoOverOne"] = Math.Round(two / one, 2), ["nOverOne"] = Math.Round(many / one, 2), ["pass"] = ok });
116150
}
117151
print("");
118152
print(pass ? $"Scaling check passed: every inline row scales at least {threshold:0.0#}x from 1 to {workers} workers."
119153
: $"Scaling check FAILED: an inline row scales less than {threshold:0.0#}x from 1 to {workers} workers; look for a process-wide serialization point on the ProcessAsync path.");
120154
print("The 2/1 ratio is diagnostic (the lock was already visible at two workers, about 1.3); it is not gated.");
155+
if (report != null)
156+
{
157+
report["workers"] = workers;
158+
report["threshold"] = threshold;
159+
report["runs"] = runs;
160+
report["counts"] = BenchResults.Numbers(counts.Select(c => (double)c));
161+
report["rows"] = cells;
162+
report["gate"] = gate;
163+
report["verdict"] = pass ? "pass" : "fail";
164+
}
121165
return pass;
122166
}
123167

@@ -128,12 +172,14 @@ internal static async Task<bool> ScaleAsync(Action<string> print, double seconds
128172
/// per request at one worker as <see cref="RunAsync"/> reports them, so a regression in one serializer's path or in the
129173
/// suspending path shows up on its own. The process-wide serializer is switched with
130174
/// <see cref="Config.SetSerializer(JsonRpcSerializer)"/> for each block and restored afterwards. A failure is printed
131-
/// after the rows measured so far and is not thrown, so it cannot change the gate's exit code.
175+
/// after the rows measured so far and is not thrown, so it cannot change the gate's exit code. When
176+
/// <paramref name="report"/> is given, the rows measured and any failure are added to it for the results file.
132177
/// </summary>
133-
internal static async Task ScaleDiagnosticsAsync(Action<string> print, double seconds, int workers)
178+
internal static async Task ScaleDiagnosticsAsync(Action<string> print, double seconds, int workers, JsonObject report = null)
134179
{
135180
var serializers = new Func<JsonRpcSerializer>[] { () => JsmnSerializer.Instance, () => new SystemTextJsonRpcSerializer(), () => new NewtonsoftJsonRpcSerializer() };
136181
var rows = new List<string>();
182+
var diagnostics = new JsonArray();
137183
string where = "start";
138184
Exception failure = null;
139185
var previous = Config.Serializer;
@@ -164,6 +210,12 @@ internal static async Task ScaleDiagnosticsAsync(Action<string> print, double se
164210
var many = await Measure(session, rowInputs, workers, seconds, processWide: yield);
165211
print($" {where}, {workers,2} workers: {many.RpcPerSec,14:N0} RPC/s ({many.Count:N0} RPCs in {many.Seconds:F3} s); {many.Cpu.TrailingText}");
166212
rows.Add($"| {shape} / {flow} | {one.RpcPerSec:N0} | {many.RpcPerSec:N0} | {many.RpcPerSec / one.RpcPerSec:F2} | {one.BytesPerRpc:F0} | {many.Cpu.TrailingText} |");
213+
diagnostics.Add(new JsonObject
214+
{
215+
["id"] = RowId("scale-diag-" + name, shape, flow), ["serializer"] = name, ["shape"] = shape, ["flow"] = flow.ToString(),
216+
["one"] = Math.Round(one.RpcPerSec), ["many"] = Math.Round(many.RpcPerSec), ["ratio"] = Math.Round(many.RpcPerSec / one.RpcPerSec, 2),
217+
["bytesPerRpc"] = Math.Round(one.BytesPerRpc), ["cpu"] = BenchResults.Cpu(many.Cpu),
218+
});
167219
}
168220
finally { Handler.DestroySession(session); }
169221
}
@@ -179,6 +231,11 @@ internal static async Task ScaleDiagnosticsAsync(Action<string> print, double se
179231
print("");
180232
if (failure != null)
181233
print($"Diagnostics stopped at {where}: {failure.GetType().Name}: {failure.Message}. The gate result above stands; the diagnostics are not part of it.");
234+
if (report != null)
235+
{
236+
report["diagnostics"] = diagnostics;
237+
if (failure != null) report["diagnosticsError"] = $"{where}: {failure.GetType().Name}: {failure.Message}";
238+
}
182239
}
183240

184241
private static double Median(List<double> values)
@@ -204,8 +261,10 @@ private static ReadOnlyMemory<byte>[] Inputs(string shape) => shape == "yield"
204261

205262
/// <summary>Checks every response, warms the row, and prints the per-shape allocation of an inline document.</summary>
206263
/// <param name="report">False checks, warms and asserts inline completion without printing the per-shape lines.</param>
207-
private static async Task ValidateAndReportAllocations(Action<string> print, string session, string shape, RpcContextFlow flow, ReadOnlyMemory<byte>[] rowInputs, bool report = true)
264+
/// <returns>Bytes per request for each input, as printed; null for the <c>yield</c> row, which is not measured per shape.</returns>
265+
private static async Task<double[]> ValidateAndReportAllocations(Action<string> print, string session, string shape, RpcContextFlow flow, ReadOnlyMemory<byte>[] rowInputs, bool report = true)
208266
{
267+
var perShape = shape == "yield" ? null : new double[rowInputs.Length];
209268
var expected = shape == "yield"
210269
? new[] { "{\"jsonrpc\":\"2.0\",\"result\":7,\"id\":6}" }
211270
: BenchmarkRunner.taskInputs.Select(t => JsonRpcProcessor.ProcessSync(t)).ToArray();
@@ -227,7 +286,9 @@ private static async Task ValidateAndReportAllocations(Action<string> print, str
227286
}
228287
long totalBytes = GC.GetAllocatedBytesForCurrentThread() - before;
229288
if (report) print($"{shape} / {flow} / shape {i + 1}: {totalBytes} B total over 2000 requests ({totalBytes / 2000.0:F1} B/RPC), including method allocations.");
289+
perShape[i] = Math.Round(totalBytes / 2000.0, 1);
230290
}
291+
return perShape;
231292
}
232293

233294
/// <summary>

‎TestServer_Console/BenchResults.cs‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.IO;
4+
using System.Linq;
5+
using System.Runtime;
6+
using System.Runtime.InteropServices;
7+
using System.Text;
8+
using System.Text.Json;
9+
using System.Text.Json.Nodes;
10+
using AustinHarris.JsonRpc;
11+
12+
namespace TestServer_Console;
13+
14+
/// <summary>
15+
/// Machine-readable results for benchmarks/charts/ingest.py. When <c>JSONRPC_BENCH_RESULTS=&lt;dir&gt;</c> is set, each
16+
/// published mode writes <c>&lt;dir&gt;/&lt;stem&gt;-&lt;run&gt;.json</c> beside its text output, with the run number from
17+
/// <c>JSONRPC_BENCH_RUN</c> (1 when unset) and the commit from <c>JSONRPC_BENCH_COMMIT</c>. Every row carries the id of the
18+
/// benchmarks.json series it feeds (or an id ingest.py lists as unpublished), its RPC/s and the CPU of its timed window.
19+
/// Nothing here runs inside a timed loop.
20+
/// </summary>
21+
internal static class BenchResults
22+
{
23+
/// <summary>The output directory, or null when results are not requested.</summary>
24+
internal static string Directory
25+
{
26+
get
27+
{
28+
var dir = Environment.GetEnvironmentVariable("JSONRPC_BENCH_RESULTS");
29+
return string.IsNullOrWhiteSpace(dir) ? null : dir;
30+
}
31+
}
32+
33+
/// <summary>True when <c>JSONRPC_BENCH_RESULTS</c> is set.</summary>
34+
internal static bool Enabled => Directory != null;
35+
36+
/// <summary>The run number for the file name: <c>JSONRPC_BENCH_RUN</c>, or 1.</summary>
37+
internal static int Run => int.TryParse(Environment.GetEnvironmentVariable("JSONRPC_BENCH_RUN"), out int run) && run > 0 ? run : 1;
38+
39+
/// <summary>The runtime as the results record it: framework, build configuration and GC mode.</summary>
40+
internal static string Runtime =>
41+
#if DEBUG
42+
$"{RuntimeInformation.FrameworkDescription}, Debug, {(GCSettings.IsServerGC ? "Server GC" : "Workstation GC")}";
43+
#else
44+
$"{RuntimeInformation.FrameworkDescription}, Release, {(GCSettings.IsServerGC ? "Server GC" : "Workstation GC")}";
45+
#endif
46+
47+
/// <summary>A results document with the fields every mode shares; the caller adds its parameters and rows.</summary>
48+
internal static JsonObject Document(string mode, double seconds) => new JsonObject
49+
{
50+
["mode"] = mode,
51+
["machine"] = MachineDescription.Current(),
52+
["runtime"] = Runtime,
53+
["serializer"] = Config.Serializer.Name,
54+
["date"] = DateTime.UtcNow.ToString("yyyy-MM-dd"),
55+
["seconds"] = seconds,
56+
["commit"] = Environment.GetEnvironmentVariable("JSONRPC_BENCH_COMMIT") is { Length: > 0 } commit ? commit : null,
57+
["run"] = Run,
58+
["cores"] = Environment.ProcessorCount,
59+
};
60+
61+
/// <summary>One result row: its series id, RPC/s rounded to a whole request, the request count and the CPU of the window.</summary>
62+
internal static JsonObject Row(string id, double rpcPerSec, long rpcs, CpuUsage cpu)
63+
{
64+
var row = new JsonObject { ["id"] = id, ["rpcPerSec"] = Math.Round(rpcPerSec), ["rpcs"] = rpcs };
65+
var usage = Cpu(cpu);
66+
if (usage.Count > 0) row["cpu"] = usage;
67+
return row;
68+
}
69+
70+
/// <summary>The CPU percentages that were measured, to one decimal; a figure the platform cannot provide is left out.</summary>
71+
internal static JsonObject Cpu(CpuUsage cpu)
72+
{
73+
var o = new JsonObject();
74+
if (cpu.SystemPercent is double system) o["system"] = Math.Round(system, 1);
75+
if (cpu.ProcessPercent is double process) o["process"] = Math.Round(process, 1);
76+
if (cpu.ClientPercent is double clients) o["clients"] = Math.Round(clients, 1);
77+
return o;
78+
}
79+
80+
/// <summary>The path a mode's file goes to: <c>&lt;dir&gt;/&lt;stem&gt;-&lt;run&gt;.json</c>.</summary>
81+
internal static string PathFor(string stem) => Path.Combine(Directory, $"{stem}-{Run}.json");
82+
83+
/// <summary>Writes the document as indented UTF-8 JSON without a byte order mark and prints the path.</summary>
84+
internal static void Write(string stem, JsonObject document, Action<string> print, string path = null)
85+
{
86+
path ??= PathFor(stem);
87+
var dir = Path.GetDirectoryName(Path.GetFullPath(path));
88+
if (!string.IsNullOrEmpty(dir)) System.IO.Directory.CreateDirectory(dir);
89+
var json = document.ToJsonString(new JsonSerializerOptions { WriteIndented = true, Encoder = System.Text.Encodings.Web.JavaScriptEncoder.UnsafeRelaxedJsonEscaping });
90+
File.WriteAllText(path, json + "\n", new UTF8Encoding(false));
91+
print?.Invoke($"wrote {path}");
92+
}
93+
94+
/// <summary>A JSON array of numbers.</summary>
95+
internal static JsonArray Numbers(IEnumerable<double> values) => new JsonArray(values.Select(v => (JsonNode)JsonValue.Create(v)).ToArray());
96+
}

0 commit comments

Comments
 (0)