From 7e82d6ce83b69b8699ff76c8fcbed1b8e392ab32 Mon Sep 17 00:00:00 2001 From: glennmichael123 Date: Wed, 30 Sep 2026 20:22:32 +0800 Subject: [PATCH 1/2] feat: define versioned JSON event protocol --- src/event_protocol.zig | 136 ++++++++++++++++++++++++++++++++++++++++ src/lib.zig | 3 + src/reporter.zig | 102 +++++++++++++----------------- src/test_history.zig | 101 +++++++++++++++++++---------- src/test_loader.zig | 54 ++++++++++------ src/ui/index.html | 14 ++--- src/ui_server.zig | 67 ++++++++++---------- tests/reporter_test.zig | 31 ++++++--- 8 files changed, 348 insertions(+), 160 deletions(-) create mode 100644 src/event_protocol.zig diff --git a/src/event_protocol.zig b/src/event_protocol.zig new file mode 100644 index 0000000..12f2b2f --- /dev/null +++ b/src/event_protocol.zig @@ -0,0 +1,136 @@ +const std = @import("std"); + +/// Major version of the public JSON event protocol. +pub const version: u16 = 1; + +pub const Status = enum { pending, running, passed, flaky, failed, skipped }; +pub const HookKind = enum { before_all, before_each, after_each, after_all }; +pub const OutputStream = enum { stdout, stderr }; + +pub const RunStart = struct { total: usize }; +pub const RunEnd = struct { + total: usize, + passed: usize, + flaky: usize, + failed: usize, + skipped: usize, + duration_ns: u64 = 0, +}; +pub const Suite = struct { name: []const u8 }; +pub const TestStart = struct { name: []const u8, suite: ?[]const u8 = null }; +pub const TestEnd = struct { + name: []const u8, + suite: ?[]const u8 = null, + status: Status, + duration_ns: u64, + error_message: ?[]const u8 = null, +}; +pub const Hook = struct { + kind: HookKind, + suite: ?[]const u8 = null, + test_name: ?[]const u8 = null, + status: Status, + duration_ns: u64 = 0, + error_message: ?[]const u8 = null, +}; +pub const Output = struct { + stream: OutputStream, + text: []const u8, + test_name: ?[]const u8 = null, +}; +pub const Retry = struct { + name: []const u8, + attempt: usize, + repetition: usize, + status: Status, + duration_ns: u64, + error_message: ?[]const u8 = null, +}; +pub const Coverage = struct { + line_percent: ?f64 = null, + function_percent: ?f64 = null, + branch_percent: ?f64 = null, + report_path: ?[]const u8 = null, +}; + +/// Every serialized value is an envelope with the same version/type/data +/// shape. Unknown fields and event types are intentionally safe to ignore. +pub const Event = union(enum) { + run_start: RunStart, + run_end: RunEnd, + suite_start: Suite, + suite_end: Suite, + test_start: TestStart, + test_end: TestEnd, + hook: Hook, + output: Output, + retry: Retry, + coverage: Coverage, + + pub fn eventName(self: Event) []const u8 { + return @tagName(std.meta.activeTag(self)); + } + + pub fn jsonStringify(self: Event, json: anytype) !void { + try json.beginObject(); + try json.objectField("protocol_version"); + try json.write(version); + try json.objectField("type"); + try json.write(self.eventName()); + try json.objectField("data"); + switch (self) { + inline else => |payload| try json.write(payload), + } + try json.endObject(); + } +}; + +pub fn status(value: anytype) Status { + return std.meta.stringToEnum(Status, @tagName(value)) orelse unreachable; +} + +pub fn encodeAlloc(allocator: std.mem.Allocator, event: Event) ![]u8 { + return std.json.Stringify.valueAlloc(allocator, event, .{}); +} + +test "every event type follows the versioned envelope contract" { + const events = [_]Event{ + .{ .run_start = .{ .total = 2 } }, + .{ .run_end = .{ .total = 2, .passed = 1, .flaky = 0, .failed = 1, .skipped = 0 } }, + .{ .suite_start = .{ .name = "suite \"one\"" } }, + .{ .suite_end = .{ .name = "suite \"one\"" } }, + .{ .test_start = .{ .name = "line\nbreak", .suite = "suite" } }, + .{ .test_end = .{ .name = "line\nbreak", .status = .failed, .duration_ns = 42, .error_message = "bad \"value\"" } }, + .{ .hook = .{ .kind = .before_each, .test_name = "case", .status = .passed } }, + .{ .output = .{ .stream = .stderr, .text = "hello\nworld", .test_name = "case" } }, + .{ .retry = .{ .name = "case", .attempt = 2, .repetition = 1, .status = .passed, .duration_ns = 10 } }, + .{ .coverage = .{ .line_percent = 90.5, .report_path = "coverage/index.html" } }, + }; + + inline for (events) |event| { + const encoded = try encodeAlloc(std.testing.allocator, event); + defer std.testing.allocator.free(encoded); + var parsed = try std.json.parseFromSlice(std.json.Value, std.testing.allocator, encoded, .{}); + defer parsed.deinit(); + const object = parsed.value.object; + try std.testing.expectEqual(@as(i64, version), object.get("protocol_version").?.integer); + try std.testing.expectEqualStrings(event.eventName(), object.get("type").?.string); + try std.testing.expect(object.get("data") != null); + } +} + +test "event payload strings use standard JSON escaping" { + const encoded = try encodeAlloc(std.testing.allocator, .{ .test_end = .{ + .name = "case\tname", + .status = .failed, + .duration_ns = 1, + .error_message = "line one\n\"quoted\" and \\escaped", + } }); + defer std.testing.allocator.free(encoded); + + var parsed = try std.json.parseFromSlice(std.json.Value, std.testing.allocator, encoded, .{}); + defer parsed.deinit(); + const data = parsed.value.object.get("data").?.object; + try std.testing.expectEqualStrings("case\tname", data.get("name").?.string); + try std.testing.expectEqualStrings("line one\n\"quoted\" and \\escaped", data.get("error_message").?.string); +} diff --git a/src/lib.zig b/src/lib.zig index d21ad13..fe23ba1 100644 --- a/src/lib.zig +++ b/src/lib.zig @@ -6,6 +6,7 @@ pub const suite = @import("suite.zig"); pub const test_runner = @import("test_runner.zig"); pub const reporter = @import("reporter.zig"); pub const pipeline = @import("pipeline.zig"); +pub const event_protocol = @import("event_protocol.zig"); pub const matchers = @import("matchers.zig"); pub const mock = @import("mock.zig"); pub const cli = @import("cli.zig"); @@ -90,6 +91,8 @@ pub const PlanOrigin = pipeline.PlanOrigin; pub const ExecutionPolicy = pipeline.ExecutionPolicy; pub const LifecycleEvent = pipeline.LifecycleEvent; pub const EventStream = pipeline.EventStream; +pub const ProtocolEvent = event_protocol.Event; +pub const protocol_version = event_protocol.version; // Matchers pub const Matchers = matchers.Matchers; diff --git a/src/reporter.zig b/src/reporter.zig index bbe375c..1b15b96 100644 --- a/src/reporter.zig +++ b/src/reporter.zig @@ -1,6 +1,7 @@ const std = @import("std"); const suite = @import("suite.zig"); const compat = @import("compat.zig"); +const protocol = @import("event_protocol.zig"); pub const ReporterType = enum { spec, @@ -464,7 +465,7 @@ pub const JsonReporter = struct { writer: std.Io.Writer, writer_ref: ?*std.Io.Writer = null, suites: std.ArrayList([]const u8), - emitted_tests: bool = false, + emitted_events: bool = false, const Self = @This(); @@ -508,87 +509,72 @@ pub const JsonReporter = struct { fn onRunStart(reporter: *Reporter, total_tests: usize) !void { const s = self(reporter); - try s.output().print("{{\"totalTests\":{d},\"tests\":[\n", .{total_tests}); + s.emitted_events = false; + s.suites.clearRetainingCapacity(); + try s.output().writeAll("[\n"); + try s.emit(.{ .run_start = .{ .total = total_tests } }); } fn onRunEnd(reporter: *Reporter, results: *TestResults) !void { const s = self(reporter); - const total_time_ms = @as(f64, @floatFromInt(results.total_time_ns)) / 1_000_000.0; - - try s.output().print("\n],\"summary\":{{\"total\":{d},\"passed\":{d},\"flaky\":{d},\"failed\":{d},\"skipped\":{d},\"time\":{d:.2}}}}}\n", .{ - results.total, - results.passed, - results.flaky, - results.failed, - results.skipped, - total_time_ms, - }); + try s.emit(.{ .run_end = .{ + .total = results.total, + .passed = results.passed, + .flaky = results.flaky, + .failed = results.failed, + .skipped = results.skipped, + .duration_ns = results.total_time_ns, + } }); + try s.output().writeAll("\n]\n"); } fn onSuiteStart(reporter: *Reporter, suite_name: []const u8) !void { const s = self(reporter); try s.suites.append(reporter.allocator, suite_name); + try s.emit(.{ .suite_start = .{ .name = suite_name } }); } fn onSuiteEnd(reporter: *Reporter, suite_name: []const u8) !void { - _ = suite_name; const s = self(reporter); + try s.emit(.{ .suite_end = .{ .name = suite_name } }); if (s.suites.items.len > 0) { _ = s.suites.pop(); } } fn onTestStart(reporter: *Reporter, test_name: []const u8) !void { - _ = reporter; - _ = test_name; + const s = self(reporter); + try s.emit(.{ .test_start = .{ + .name = test_name, + .suite = if (s.suites.getLastOrNull()) |name| name else null, + } }); } fn onTestEnd(reporter: *Reporter, test_case: *const suite.TestCase) !void { const s = self(reporter); - const time_ms = @as(f64, @floatFromInt(test_case.execution_time_ns)) / 1_000_000.0; - - const status_str = switch (test_case.status) { - .passed => "passed", - .flaky => "flaky", - .failed => "failed", - .skipped => "skipped", - else => "unknown", - }; - - if (s.emitted_tests) try s.output().print(",\n", .{}); - s.emitted_tests = true; - - // Note: In a real implementation, you'd want to properly escape JSON strings - try s.output().print(" {{\"name\":\"{s}\",\"status\":\"{s}\",\"time\":{d:.2}", .{ - test_case.name, - status_str, - time_ms, - }); - - if (test_case.error_message) |msg| { - try s.output().print(",\"error\":\"{s}\"", .{msg}); - } - - try s.output().print(",\"attempts\":[", .{}); - for (test_case.attempts.items, 0..) |attempt, index| { - if (index > 0) try s.output().print(",", .{}); - const attempt_status = switch (attempt.status) { - .passed => "passed", - .flaky => "flaky", - .failed => "failed", - .skipped => "skipped", - else => "unknown", - }; - try s.output().print( - "{{\"number\":{d},\"repetition\":{d},\"status\":\"{s}\",\"durationNs\":{d}", - .{ attempt.number, attempt.repetition, attempt_status, attempt.duration_ns }, - ); - if (attempt.error_message) |message| { - try s.output().print(",\"error\":\"{s}\"", .{message}); - } - try s.output().print("}}", .{}); + for (test_case.attempts.items) |attempt| { + try s.emit(.{ .retry = .{ + .name = test_case.name, + .attempt = attempt.number, + .repetition = attempt.repetition, + .status = protocol.status(attempt.status), + .duration_ns = attempt.duration_ns, + .error_message = attempt.error_message, + } }); } - try s.output().print("]}}", .{}); + try s.emit(.{ .test_end = .{ + .name = test_case.name, + .suite = if (s.suites.getLastOrNull()) |name| name else null, + .status = protocol.status(test_case.status), + .duration_ns = test_case.execution_time_ns, + .error_message = test_case.error_message, + } }); + } + + fn emit(s: *Self, event: protocol.Event) !void { + if (s.emitted_events) try s.output().writeAll(",\n"); + s.emitted_events = true; + try std.json.Stringify.value(event, .{}, s.output()); } }; diff --git a/src/test_history.zig b/src/test_history.zig index 93d31ae..91ac0b1 100644 --- a/src/test_history.zig +++ b/src/test_history.zig @@ -2,12 +2,14 @@ const std = @import("std"); const reporter_mod = @import("reporter.zig"); const suite = @import("suite.zig"); const compat = @import("compat.zig"); +const protocol = @import("event_protocol.zig"); /// Test history entry pub const HistoryEntry = struct { timestamp: i64, total: usize, passed: usize, + flaky: usize = 0, failed: usize, skipped: usize, duration_ns: u64, @@ -69,6 +71,7 @@ pub const TestHistory = struct { .timestamp = self.start_time, .total = total, .passed = 0, + .flaky = 0, .failed = 0, .skipped = 0, .duration_ns = 0, @@ -111,6 +114,7 @@ pub const TestHistory = struct { const end_time = compat.milliTimestamp(); self.current_entry.?.duration_ns = @intCast((end_time - self.start_time) * std.time.ns_per_ms); self.current_entry.?.passed = results.passed; + self.current_entry.?.flaky = results.flaky; self.current_entry.?.failed = results.failed; self.current_entry.?.skipped = results.skipped; @@ -128,42 +132,36 @@ pub const TestHistory = struct { var filename_buf: [256]u8 = undefined; const filename = try std.fmt.bufPrint(&filename_buf, "{s}/test-run-{d}.json", .{ self.history_dir, self.current_entry.?.timestamp }); - // Write JSON + // Store the same versioned events consumed by reporters and the UI. var buffer = std.ArrayList(u8).empty; defer buffer.deinit(self.allocator); - - try buffer.appendSlice(self.allocator, "{\n"); - try buffer.print(self.allocator, " \"timestamp\": {d},\n", .{self.current_entry.?.timestamp}); - try buffer.print(self.allocator, " \"total\": {d},\n", .{self.current_entry.?.total}); - try buffer.print(self.allocator, " \"passed\": {d},\n", .{self.current_entry.?.passed}); - try buffer.print(self.allocator, " \"failed\": {d},\n", .{self.current_entry.?.failed}); - try buffer.print(self.allocator, " \"skipped\": {d},\n", .{self.current_entry.?.skipped}); - try buffer.print(self.allocator, " \"duration_ns\": {d},\n", .{self.current_entry.?.duration_ns}); - try buffer.appendSlice(self.allocator, " \"tests\": [\n"); - - for (self.current_entry.?.tests.items, 0..) |test_record, i| { - try buffer.appendSlice(self.allocator, " {\n"); - try buffer.print(self.allocator, " \"name\": \"{s}\",\n", .{test_record.name}); - try buffer.print(self.allocator, " \"suite_name\": \"{s}\",\n", .{test_record.suite_name}); - try buffer.print(self.allocator, " \"status\": \"{s}\",\n", .{test_record.status}); - try buffer.print(self.allocator, " \"execution_time_ns\": {d}", .{test_record.execution_time_ns}); - - if (test_record.error_message) |msg| { - try buffer.appendSlice(self.allocator, ",\n"); - try buffer.print(self.allocator, " \"error_message\": \"{s}\"\n", .{msg}); - } else { - try buffer.appendSlice(self.allocator, "\n"); - } - - if (i < self.current_entry.?.tests.items.len - 1) { - try buffer.appendSlice(self.allocator, " },\n"); - } else { - try buffer.appendSlice(self.allocator, " }\n"); - } + var allocating: std.Io.Writer.Allocating = .fromArrayList(self.allocator, &buffer); + errdefer buffer = allocating.toArrayList(); + const writer = &allocating.writer; + try writer.writeAll("[\n"); + try std.json.Stringify.value(protocol.Event{ .run_start = .{ .total = self.current_entry.?.total } }, .{}, writer); + for (self.current_entry.?.tests.items) |test_record| { + try writer.writeAll(",\n"); + try std.json.Stringify.value(protocol.Event{ .test_end = .{ + .name = test_record.name, + .suite = test_record.suite_name, + .status = std.meta.stringToEnum(protocol.Status, test_record.status) orelse .failed, + .duration_ns = test_record.execution_time_ns, + .error_message = test_record.error_message, + } }, .{}, writer); } - - try buffer.appendSlice(self.allocator, " ]\n"); - try buffer.appendSlice(self.allocator, "}\n"); + try writer.writeAll(",\n"); + try std.json.Stringify.value(protocol.Event{ .run_end = .{ + .total = self.current_entry.?.total, + .passed = self.current_entry.?.passed, + .flaky = self.current_entry.?.flaky, + .failed = self.current_entry.?.failed, + .skipped = self.current_entry.?.skipped, + .duration_ns = self.current_entry.?.duration_ns, + } }, .{}, writer); + try writer.writeAll("\n]\n"); + try writer.flush(); + buffer = allocating.toArrayList(); try compat.writeFile(self.allocator, filename, buffer.items); } @@ -190,6 +188,7 @@ pub const TestHistory = struct { .timestamp = 0, .total = 0, .passed = 0, + .flaky = 0, .failed = 0, .skipped = 0, .duration_ns = 0, @@ -249,3 +248,39 @@ test "TestHistory start and record" { try history.recordTest("test1", "suite1", .passed, 1000000, null); try std.testing.expectEqual(@as(usize, 1), history.current_entry.?.tests.items.len); } + +test "TestHistory saves versioned protocol events" { + const allocator = std.testing.allocator; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const history_dir = try std.fmt.allocPrint(allocator, ".zig-cache/tmp/{s}", .{tmp.sub_path}); + defer allocator.free(history_dir); + + var history = TestHistory.init(allocator, history_dir); + defer history.deinit(); + try history.startRun(1); + try history.recordTest("quoted \"test\"", "suite\nname", .passed, 42, null); + var results = reporter_mod.TestResults.init(allocator); + defer results.deinit(); + results.total = 1; + results.passed = 1; + results.total_time_ns = 42; + try history.finishRun(&results); + + const filename = try std.fmt.allocPrint(allocator, "{s}/test-run-{d}.json", .{ history_dir, history.current_entry.?.timestamp }); + defer allocator.free(filename); + const content = try compat.readFileAlloc(allocator, filename); + defer allocator.free(content); + var parsed = try std.json.parseFromSlice(std.json.Value, allocator, content, .{}); + defer parsed.deinit(); + + const events = parsed.value.array.items; + try std.testing.expectEqual(@as(usize, 3), events.len); + try std.testing.expectEqualStrings("run_start", events[0].object.get("type").?.string); + try std.testing.expectEqualStrings("test_end", events[1].object.get("type").?.string); + try std.testing.expectEqualStrings("quoted \"test\"", events[1].object.get("data").?.object.get("name").?.string); + try std.testing.expectEqualStrings("run_end", events[2].object.get("type").?.string); + for (events) |event| { + try std.testing.expectEqual(@as(i64, protocol.version), event.object.get("protocol_version").?.integer); + } +} diff --git a/src/test_loader.zig b/src/test_loader.zig index bb44757..19fc984 100644 --- a/src/test_loader.zig +++ b/src/test_loader.zig @@ -7,6 +7,7 @@ const suite = @import("suite.zig"); const coverage = @import("coverage.zig"); const ui_server = @import("ui_server.zig"); const compat = @import("compat.zig"); +const protocol = @import("event_protocol.zig"); /// Options for running discovered tests pub const LoaderOptions = struct { @@ -41,12 +42,6 @@ pub const LoaderOptions = struct { ui_server: ?*ui_server.UIServer = null, }; -fn broadcastJson(server: *ui_server.UIServer, allocator: std.mem.Allocator, event: []const u8, value: anytype) !void { - const json = try std.json.Stringify.valueAlloc(allocator, value, .{}); - defer allocator.free(json); - try server.broadcast(event, json); -} - /// Run all discovered test files pub fn runDiscoveredTests( allocator: std.mem.Allocator, @@ -107,7 +102,7 @@ pub fn runDiscoveredTests( // Notify UI of run start if (options.ui_server) |server| { - try broadcastJson(server, allocator, "run_start", .{ .total = selected_files }); + try server.broadcastEvent(allocator, .{ .run_start = .{ .total = selected_files } }); } var files_run: usize = 0; @@ -129,8 +124,8 @@ pub fn runDiscoveredTests( // Notify UI of test file start if (options.ui_server) |server| { - try broadcastJson(server, allocator, "suite_start", .{ .name = file.name }); - try broadcastJson(server, allocator, "test_start", .{ .name = file.name }); + try server.broadcastEvent(allocator, .{ .suite_start = .{ .name = file.name } }); + try server.broadcastEvent(allocator, .{ .test_start = .{ .name = file.name, .suite = file.name } }); } files_run += 1; @@ -182,14 +177,24 @@ pub fn runDiscoveredTests( // Notify UI of test file end if (options.ui_server) |server| { - try broadcastJson(server, allocator, "test_end", .{ + for (test_case.attempts.items) |attempt| { + try server.broadcastEvent(allocator, .{ .retry = .{ + .name = file.name, + .attempt = attempt.number, + .repetition = attempt.repetition, + .status = protocol.status(attempt.status), + .duration_ns = attempt.duration_ns, + .error_message = attempt.error_message, + } }); + } + try server.broadcastEvent(allocator, .{ .test_end = .{ .name = file.name, - .status = @tagName(test_case.status), - .execution_time_ns = test_case.execution_time_ns, - .attempts = test_case.attempts.items.len, - .error_message = "", - }); - try broadcastJson(server, allocator, "suite_end", .{ .name = file.name }); + .suite = file.name, + .status = protocol.status(test_case.status), + .duration_ns = test_case.execution_time_ns, + .error_message = test_case.error_message, + } }); + try server.broadcastEvent(allocator, .{ .suite_end = .{ .name = file.name } }); } if (test_case.status == .passed) { @@ -213,19 +218,30 @@ pub fn runDiscoveredTests( std.debug.print("Warning: Could not parse coverage report: {any}\n", .{err}); break :result null; }; - if (cov_result) |summary| coverage.printCoverageSummary(summary); + if (cov_result) |summary| { + coverage.printCoverageSummary(summary); + if (options.ui_server) |server| { + try server.broadcastEvent(allocator, .{ .coverage = .{ + .line_percent = summary.linePercentage(), + .function_percent = summary.functionPercentage(), + .branch_percent = summary.branchPercentage(), + .report_path = summary.report_dir, + } }); + } + } } } // Notify UI of run end if (options.ui_server) |server| { - try broadcastJson(server, allocator, "run_end", .{ + try server.broadcastEvent(allocator, .{ .run_end = .{ .total = results.total, .passed = results.passed, .flaky = results.flaky, .failed = results.failed, .skipped = results.skipped, - }); + .duration_ns = results.total_time_ns, + } }); } if (options.shard) |shard| { diff --git a/src/ui/index.html b/src/ui/index.html index e46f81d..fcebb70 100644 --- a/src/ui/index.html +++ b/src/ui/index.html @@ -332,26 +332,26 @@

Waiting for tests...

}); eventSource.addEventListener('run_start', (e) => { - const data = JSON.parse(e.data); + const data = JSON.parse(e.data).data; clearTests(); state.total = data.total; updateStats(); }); eventSource.addEventListener('suite_start', (e) => { - const data = JSON.parse(e.data); + const data = JSON.parse(e.data).data; state.currentSuite = data.name; addSuite(data.name); }); eventSource.addEventListener('test_start', (e) => { - const data = JSON.parse(e.data); - addTest(state.currentSuite, data.name, 'running'); + const data = JSON.parse(e.data).data; + addTest(data.suite || state.currentSuite, data.name, 'running'); }); eventSource.addEventListener('test_end', (e) => { - const data = JSON.parse(e.data); - updateTest(state.currentSuite, data.name, data.status, data.execution_time_ns, data.error_message); + const data = JSON.parse(e.data).data; + updateTest(data.suite || state.currentSuite, data.name, data.status, data.duration_ns, data.error_message || ''); if (data.status === 'passed') state.passed++; else if (data.status === 'flaky') state.flaky++; @@ -362,7 +362,7 @@

Waiting for tests...

}); eventSource.addEventListener('run_end', (e) => { - const data = JSON.parse(e.data); + const data = JSON.parse(e.data).data; state.total = data.total; state.passed = data.passed; state.flaky = data.flaky; diff --git a/src/ui_server.zig b/src/ui_server.zig index 1177c98..5cdc321 100644 --- a/src/ui_server.zig +++ b/src/ui_server.zig @@ -3,6 +3,7 @@ const reporter_mod = @import("reporter.zig"); const suite = @import("suite.zig"); const test_history = @import("test_history.zig"); const compat = @import("compat.zig"); +const protocol = @import("event_protocol.zig"); /// Options for the UI server pub const UIServerOptions = struct { @@ -282,13 +283,19 @@ pub const UIServer = struct { index += 1; } } + + /// Serialize and broadcast one versioned protocol event. + pub fn broadcastEvent(self: *Self, allocator: std.mem.Allocator, event: protocol.Event) !void { + const encoded = try protocol.encodeAlloc(allocator, event); + defer allocator.free(encoded); + try self.broadcast(event.eventName(), encoded); + } }; /// UI Reporter - sends test events to the UI server pub const UIReporter = struct { reporter: reporter_mod.Reporter, server: *UIServer, - buffer: std.ArrayList(u8), const Self = @This(); @@ -307,71 +314,66 @@ pub const UIReporter = struct { .use_colors = false, }, .server = server, - .buffer = .empty, }; } - pub fn deinit(self: *Self) void { - self.buffer.deinit(self.reporter.allocator); - } - - fn encode(self: *Self, value: anytype) !void { - self.buffer.clearRetainingCapacity(); - var writer: std.Io.Writer.Allocating = .fromArrayList(self.reporter.allocator, &self.buffer); - defer self.buffer = writer.toArrayList(); - try std.json.Stringify.value(value, .{}, &writer.writer); - } + pub fn deinit(_: *Self) void {} fn onRunStart(reporter: *reporter_mod.Reporter, total: usize) !void { const self: *Self = @fieldParentPtr("reporter", reporter); - try self.encode(.{ .total = total }); - try self.server.broadcast("run_start", self.buffer.items); + try self.server.broadcastEvent(reporter.allocator, .{ .run_start = .{ .total = total } }); } fn onRunEnd(reporter: *reporter_mod.Reporter, results: *reporter_mod.TestResults) !void { const self: *Self = @fieldParentPtr("reporter", reporter); - try self.encode(.{ + try self.server.broadcastEvent(reporter.allocator, .{ .run_end = .{ .total = results.total, .passed = results.passed, .flaky = results.flaky, .failed = results.failed, .skipped = results.skipped, - }); - try self.server.broadcast("run_end", self.buffer.items); + .duration_ns = results.total_time_ns, + } }); } fn onSuiteStart(reporter: *reporter_mod.Reporter, suite_name: []const u8) !void { const self: *Self = @fieldParentPtr("reporter", reporter); - try self.encode(.{ .name = suite_name }); - try self.server.broadcast("suite_start", self.buffer.items); + try self.server.broadcastEvent(reporter.allocator, .{ .suite_start = .{ .name = suite_name } }); } fn onSuiteEnd(reporter: *reporter_mod.Reporter, suite_name: []const u8) !void { const self: *Self = @fieldParentPtr("reporter", reporter); - try self.encode(.{ .name = suite_name }); - try self.server.broadcast("suite_end", self.buffer.items); + try self.server.broadcastEvent(reporter.allocator, .{ .suite_end = .{ .name = suite_name } }); } fn onTestStart(reporter: *reporter_mod.Reporter, test_name: []const u8) !void { const self: *Self = @fieldParentPtr("reporter", reporter); - try self.encode(.{ .name = test_name }); - try self.server.broadcast("test_start", self.buffer.items); + try self.server.broadcastEvent(reporter.allocator, .{ .test_start = .{ .name = test_name } }); } fn onTestEnd(reporter: *reporter_mod.Reporter, test_case: *const suite.TestCase) !void { const self: *Self = @fieldParentPtr("reporter", reporter); - try self.encode(.{ + for (test_case.attempts.items) |attempt| { + try self.server.broadcastEvent(reporter.allocator, .{ .retry = .{ + .name = test_case.name, + .attempt = attempt.number, + .repetition = attempt.repetition, + .status = protocol.status(attempt.status), + .duration_ns = attempt.duration_ns, + .error_message = attempt.error_message, + } }); + } + try self.server.broadcastEvent(reporter.allocator, .{ .test_end = .{ .name = test_case.name, - .status = @tagName(test_case.status), - .execution_time_ns = test_case.execution_time_ns, - .error_message = test_case.error_message orelse "", - }); - try self.server.broadcast("test_end", self.buffer.items); + .status = protocol.status(test_case.status), + .duration_ns = test_case.execution_time_ns, + .error_message = test_case.error_message, + } }); } }; @@ -471,10 +473,7 @@ test "UIReporter initialization" { defer ui_reporter.deinit(); try std.testing.expectEqual(false, ui_reporter.reporter.use_colors); - try std.testing.expectEqual(@as(usize, 0), ui_reporter.buffer.items.len); - try ui_reporter.reporter.onSuiteStart("quoted \"suite\"\nname"); - try std.testing.expectEqualStrings("{\"name\":\"quoted \\\"suite\\\"\\nname\"}", ui_reporter.buffer.items); } fn writeTestRequest(io: std.Io, stream: std.Io.net.Stream, target: []const u8) !void { @@ -535,9 +534,9 @@ test "UI server streams broadcast events over SSE" { try expectLine(&stream_reader.interface, "data: {}"); try expectLine(&stream_reader.interface, ""); - try server.broadcast("run_start", "{\"total\":2}"); + try server.broadcastEvent(std.testing.allocator, .{ .run_start = .{ .total = 2 } }); try expectLine(&stream_reader.interface, "event: run_start"); - try expectLine(&stream_reader.interface, "data: {\"total\":2}"); + try expectLine(&stream_reader.interface, "data: {\"protocol_version\":1,\"type\":\"run_start\",\"data\":{\"total\":2}}"); try expectLine(&stream_reader.interface, ""); } diff --git a/tests/reporter_test.zig b/tests/reporter_test.zig index 37b0ffe..6882e6b 100644 --- a/tests/reporter_test.zig +++ b/tests/reporter_test.zig @@ -47,8 +47,8 @@ test "reporter dot output" { try std.testing.expect(std.mem.indexOf(u8, output, "Passed: 1") != null); } -test "reporter json output" { - var buffer: [2048]u8 = undefined; +test "reporter json output uses versioned event envelopes" { + var buffer: [4096]u8 = undefined; const writer: std.Io.Writer = .fixed(&buffer); var reporter = ztf.JsonReporter.init(std.testing.allocator, writer); defer reporter.deinit(); @@ -56,20 +56,29 @@ test "reporter json output" { defer results.deinit(); results.total = 1; results.passed = 1; - var test_case = ztf.TestCase.init("passes", passingTest); + var test_case = ztf.TestCase.init("passes \"quoted\"\nname", passingTest); test_case.status = .passed; try reporter.reporter.onRunStart(1); - try reporter.reporter.onSuiteStart("suite"); + try reporter.reporter.onSuiteStart("suite\tone"); + try reporter.reporter.onTestStart(test_case.name); try reporter.reporter.onTestEnd(&test_case); - try reporter.reporter.onSuiteEnd("suite"); + try reporter.reporter.onSuiteEnd("suite\tone"); try reporter.reporter.onRunEnd(&results); const output = buffer[0..reporter.writer.end]; - try std.testing.expect(std.mem.indexOf(u8, output, "\"name\":\"passes\"") != null); - try std.testing.expect(std.mem.indexOf(u8, output, "\"passed\":1") != null); var parsed = try std.json.parseFromSlice(std.json.Value, std.testing.allocator, output, .{}); defer parsed.deinit(); + const events = parsed.value.array.items; + try std.testing.expectEqual(@as(usize, 6), events.len); + try std.testing.expectEqual(@as(i64, ztf.protocol_version), events[0].object.get("protocol_version").?.integer); + try std.testing.expectEqualStrings("run_start", events[0].object.get("type").?.string); + try std.testing.expectEqualStrings("test_start", events[2].object.get("type").?.string); + const test_data = events[3].object.get("data").?.object; + try std.testing.expectEqualStrings(test_case.name, test_data.get("name").?.string); + try std.testing.expectEqualStrings("suite\tone", test_data.get("suite").?.string); + try std.testing.expectEqualStrings("run_end", events[5].object.get("type").?.string); + try std.testing.expectEqual(@as(i64, 1), events[5].object.get("data").?.object.get("passed").?.integer); } test "reporter json output includes flaky attempt history" { @@ -106,10 +115,14 @@ test "reporter json output includes flaky attempt history" { const output = buffer[0..reporter.writer.end]; try std.testing.expect(std.mem.indexOf(u8, output, "\"status\":\"flaky\"") != null); - try std.testing.expect(std.mem.indexOf(u8, output, "\"attempts\":[") != null); - try std.testing.expect(std.mem.indexOf(u8, output, "\"durationNs\":10") != null); + try std.testing.expect(std.mem.indexOf(u8, output, "\"type\":\"retry\"") != null); + try std.testing.expect(std.mem.indexOf(u8, output, "\"duration_ns\":10") != null); var parsed = try std.json.parseFromSlice(std.json.Value, allocator, output, .{}); defer parsed.deinit(); + const events = parsed.value.array.items; + try std.testing.expectEqualStrings("retry", events[2].object.get("type").?.string); + try std.testing.expectEqualStrings("retry", events[3].object.get("type").?.string); + try std.testing.expectEqualStrings("test_end", events[4].object.get("type").?.string); } test "reporter set selects the same built-ins for every executor" { From 26b82c99595300e602db8006f3493c4bae38f25c Mon Sep 17 00:00:00 2001 From: glennmichael123 Date: Wed, 30 Sep 2026 20:22:41 +0800 Subject: [PATCH 2/2] docs: document JSON event protocol --- README.md | 33 +++++++++-------- docs/execution-pipeline.md | 7 ++-- docs/json-event-protocol.md | 71 +++++++++++++++++++++++++++++++++++++ 3 files changed, 94 insertions(+), 17 deletions(-) create mode 100644 docs/json-event-protocol.md diff --git a/README.md b/README.md index ac03ae1..6629198 100644 --- a/README.md +++ b/README.md @@ -820,28 +820,33 @@ Passed: 12, Failed: 0, Total: 12 (1.23ms) ### JSON Reporter -Machine-readable output for tooling: +Machine-readable output uses the versioned event protocol shared with the live +UI and test history: ```json -{ - "totalTests": 12, - "tests": [ - { +[ + { + "protocol_version": 1, + "type": "run_start", + "data": { "total": 12 } + }, + { + "protocol_version": 1, + "type": "test_end", + "data": { "name": "should add two numbers", + "suite": "Math operations", "status": "passed", - "time": 0.05 + "duration_ns": 50000, + "error_message": null } - ], - "summary": { - "total": 12, - "passed": 12, - "failed": 0, - "skipped": 0, - "time": 1.23 } -} +] ``` +See [docs/json-event-protocol.md](docs/json-event-protocol.md) for every event +type and the compatibility policy. + ## API Reference See [docs/api.md](docs/api.md) for complete API documentation. diff --git a/docs/execution-pipeline.md b/docs/execution-pipeline.md index 54040ed..6b9de40 100644 --- a/docs/execution-pipeline.md +++ b/docs/execution-pipeline.md @@ -43,9 +43,10 @@ New plan producers should append `PlanItem` values. New executors should emit implement `Reporter.VTable`; `EventStream` adapts the typed lifecycle to that stable callback interface. -The event types are intentionally internal Zig structures. The separately -versioned JSON wire protocol is built on this layer rather than making this API -depend on one serialization format. +The lifecycle structures remain format-independent. The JSON reporter, live UI, +and test history adapt them to the public, versioned +[JSON event protocol](json-event-protocol.md), so integrations share one wire +model without coupling the execution API to JSON. ## Migration diff --git a/docs/json-event-protocol.md b/docs/json-event-protocol.md new file mode 100644 index 0000000..ac32b57 --- /dev/null +++ b/docs/json-event-protocol.md @@ -0,0 +1,71 @@ +# JSON event protocol + +The framework exposes one versioned event model for the JSON reporter, live UI, +and saved test history. Protocol version 1 uses this envelope for every event: + +```json +{ + "protocol_version": 1, + "type": "test_end", + "data": { + "name": "adds two numbers", + "suite": "math", + "status": "passed", + "duration_ns": 42000, + "error_message": null + } +} +``` + +All strings are encoded by Zig's standard JSON serializer. Consumers should +decode JSON rather than depending on whitespace or object-field order. + +## Transports + +- The JSON reporter writes an ordered JSON array of event envelopes. +- The live UI sends each envelope as a server-sent event. The SSE event name is + the same as the envelope's `type`. +- Test-history files contain an ordered JSON array using the same envelopes. + +Events preserve execution order. A run begins with `run_start` and finishes +with `run_end`; suite and test start/end events are nested in between. Retry +events are emitted in attempt order immediately before their `test_end`. + +## Version 1 events + +| Type | Data fields | +| --- | --- | +| `run_start` | `total` | +| `run_end` | `total`, `passed`, `flaky`, `failed`, `skipped`, `duration_ns` | +| `suite_start`, `suite_end` | `name` | +| `test_start` | `name`, optional `suite` | +| `test_end` | `name`, optional `suite`, `status`, `duration_ns`, optional `error_message` | +| `hook` | `kind`, optional `suite`, optional `test_name`, `status`, `duration_ns`, optional `error_message` | +| `output` | `stream`, `text`, optional `test_name` | +| `retry` | `name`, `attempt`, `repetition`, `status`, `duration_ns`, optional `error_message` | +| `coverage` | optional `line_percent`, `function_percent`, `branch_percent`, `report_path` | + +Statuses are `pending`, `running`, `passed`, `flaky`, `failed`, or `skipped`. +Hook kinds are `before_all`, `before_each`, `after_each`, or `after_all`. +Output streams are `stdout` or `stderr`. + +Optional fields are serialized as `null` when no value is available. Durations +are unsigned integer nanoseconds. Percentages are numbers from 0 through 100. + +## Compatibility policy + +`protocol_version` is the major wire-format version. Version 1 may gain new +event types, enum values, and optional data fields without changing the version. +Consumers must ignore event types and fields they do not understand. Existing +fields will not change meaning or type within a major version. + +Removing or renaming an event or field, changing a field's type or semantics, +or changing the envelope shape requires a new major version. Producers emit one +major version per stream. Consumers should reject an unsupported major version +with a clear error instead of interpreting it as an older version. + +## Zig API + +`event_protocol.Event` is the tagged union for all event types, and +`event_protocol.encodeAlloc` returns a standards-compliant encoded envelope. +The root module also exports these as `ProtocolEvent` and `protocol_version`.