diff --git a/README.md b/README.md index 813ea84..a82eeee 100644 --- a/README.md +++ b/README.md @@ -95,15 +95,20 @@ This will automatically discover and run all `*.test.zig` files in the `tests` d - `--no-recursive` - Disable recursive directory search - `--bail` - Stop on first failure - `--filter ` / `--grep ` - Run only Zig tests whose names contain the text +- `--reporter ` - Use spec, dot, JSON, TAP, or JUnit output +- `--junit-output ` - Set the JUnit XML path +- `--timeout ` - Mark files that exceed the global time budget as failed - `--shard-index ` - Run one one-based file shard - `--shard-count ` - Set the total number of file shards - `--no-color` - Disable color in child Zig test processes - `--verbose` - Show detailed output -Discovery mode rejects options that it cannot honor yet instead of silently -ignoring them. Reporter selection, quiet output, timeouts, parallel jobs, -snapshot updates, memory profiling, JUnit output, and the web UI require the -shared execution pipeline planned for future releases. +Discovery and programmatic execution share the same test-plan, lifecycle, +result, and reporter model. Discovery still rejects options it cannot honor +instead of silently ignoring them: quiet output, parallel jobs, snapshot +updates, memory profiling, and the web UI remain programmatic-only. See +[the execution pipeline architecture](docs/execution-pipeline.md) for extension +points, timeout semantics, and migration guidance. **Examples:** @@ -634,7 +639,9 @@ zig-test --no-color | `--version` | `-v` | Show version information | | `--bail` | `-b` | Stop on first failure | | `--filter ` | | Run only tests matching pattern | -| `--reporter ` | | Set reporter (spec, dot, json) | +| `--reporter ` | | Set reporter (spec, dot, json, tap, junit) | +| `--junit-output ` | | Set the JUnit XML output path | +| `--timeout ` | | Set the global test timeout | | `--verbose` | | Enable verbose output | | `--quiet` | `-q` | Minimal output | | `--no-color` | | Disable colored output | diff --git a/build.zig b/build.zig index 8adea63..3a0ab38 100644 --- a/build.zig +++ b/build.zig @@ -286,6 +286,13 @@ pub fn build(b: *std.Build) void { run_shard_two.addArgs(&.{ "--test-dir", "tests", "--pattern", "sample.test.zig", "--shard-index", "2", "--shard-count", "2", "--no-color" }); test_step.dependOn(&run_shard_two.step); + const run_discovery_tap = b.addRunArtifact(exe); + run_discovery_tap.addArgs(&.{ "--test-dir", "tests", "--pattern", "sample.test.zig", "--reporter", "tap", "--no-color" }); + run_discovery_tap.expectStdOutEqual( + "TAP version 14\n1..1\n# Subtest: sample.test.zig\nok 1 - sample.test.zig\n", + ); + test_step.dependOn(&run_discovery_tap.step); + const run_version = b.addRunArtifact(exe); run_version.addArg("--version"); run_version.expectStdErrEqual(b.fmt("Zig Test Framework v{s}\n", .{package_version})); diff --git a/docs/execution-pipeline.md b/docs/execution-pipeline.md new file mode 100644 index 0000000..239758a --- /dev/null +++ b/docs/execution-pipeline.md @@ -0,0 +1,62 @@ +# Unified execution pipeline + +The framework has two plan producers and two executor implementations, but one +execution contract. + +1. Programmatic registration or file discovery produces a `TestPlan`. +2. A registered-test or external-process executor consumes that plan. +3. The executor emits typed `LifecycleEvent` values through `EventStream`. +4. A selected reporter consumes the events and a shared `TestResults` summary. + +`PlanItem.registered` retains the suite and test-case pointers needed for hooks +and in-process execution. `PlanItem.discovered` retains the discovered file used +by the external `zig test` executor. `PlanItem.identity()` gives extensions a +common stable identity without erasing executor-specific information. + +## Shared lifecycle and results + +Both executors emit run, suite, and test start/end events. Discovered files are +represented as one synthetic suite and test result per external process. This +means spec, dot, JSON, TAP, and JUnit reporters receive the same lifecycle and +summary model in either mode. `ReporterSet` is the single factory for built-in +reporters, including the configured JUnit output path. + +`ExecutionPolicy` owns common filtering, bail, and timeout decisions: + +- Registered tests match filters directly by test name. Discovery forwards the + same filter to `zig test`, where Zig applies it to names inside each file. +- Bail stops scheduling after the shared result model records a failure. Work + already started by a parallel batch cannot be cancelled. +- A global timeout compares elapsed execution time with the same policy in both + executors. It marks an over-budget result failed after control returns; it + does not yet terminate an in-process function or external child process. +- Summaries always come from `TestResults`, rather than executor-local counters. + +## Extension points + +New plan producers should append `PlanItem` values. New executors should emit +`LifecycleEvent` values and update `TestResults`. New reporters can continue to +implement `Reporter.VTable`; `EventStream` adapts the typed lifecycle to that +stable callback interface. + +The event types are intentionally internal Zig structures. The separately +versioned JSON wire protocol is built on this layer rather than making this API +depend on one serialization format. + +## Migration + +Existing programmatic APIs remain source compatible: + +- `TestRunner.init`, `runTests`, and `runTestsWithOptions` are unchanged. +- `RunnerOptions` only gains optional `junit_output`, `timeout_ms`, and + `reporter_writer` fields. +- `runDiscoveredTests` and `LoaderOptions` remain available; optional reporter, + JUnit path, timeout, and writer fields default to the previous spec behavior. + +CLI hosts can inject a stdout writer with `reporter_writer`. Embedded callers +may leave it unset; reporter output then uses stderr so it cannot corrupt Zig's +stdout-based test protocol. + +Code that implements a custom execution path can migrate incrementally by first +building a `TestPlan`, then routing callbacks through `EventStream`. Existing +custom reporters require no migration. diff --git a/src/cli.zig b/src/cli.zig index 66aec1a..8735d14 100644 --- a/src/cli.zig +++ b/src/cli.zig @@ -326,7 +326,6 @@ pub const CLI = struct { if (!self.options.watch and self.options.watch_debounce != 300) return "--watch-debounce"; if (!self.options.ui and self.options.ui_port != 8080) return "--ui-port"; if (!self.options.ui and !std.mem.eql(u8, self.options.ui_host, "127.0.0.1")) return "--ui-host"; - if (self.options.reporter != .spec) return "--reporter"; if (self.options.quiet) return "--quiet"; if (self.options.parallel) return "--parallel"; if (self.options.jobs != null) return "--jobs"; @@ -335,8 +334,6 @@ pub const CLI = struct { if (self.options.profile_memory) return "--profile-memory"; if (self.options.memory_threshold != 0) return "--memory-threshold"; if (self.options.fail_on_leak) return "--fail-on-leak"; - if (self.options.junit_output != null) return "--junit-output"; - if (self.options.timeout != null) return "--timeout"; if (self.options.ui) return "--ui"; return null; } @@ -442,6 +439,8 @@ pub const CLI = struct { .use_colors = !self.options.no_color, .parallel = self.options.parallel, .n_jobs = self.options.jobs, + .junit_output = self.options.junit_output orelse "test-results.xml", + .timeout_ms = self.options.timeout, }; } }; @@ -691,18 +690,18 @@ test "discovery accepts implemented options" { try std.testing.expectEqual(@as(?[]const u8, null), cli.unsupportedDiscoveryOption()); } -test "discovery identifies unsupported options" { +test "discovery accepts reporter selection" { var cli = CLI.init(std.testing.allocator); const args = [_][]const u8{ "zig-test", "--test-dir", "tests", "--reporter", "json" }; try cli.parse(&args); - try std.testing.expectEqualStrings("--reporter", cli.unsupportedDiscoveryOption().?); + try std.testing.expectEqual(@as(?[]const u8, null), cli.unsupportedDiscoveryOption()); } -test "discovery reports timeout and parallel options as unsupported" { +test "discovery accepts timeout and reports parallel as unsupported" { var cli = CLI.init(std.testing.allocator); cli.options.timeout = 1000; - try std.testing.expectEqualStrings("--timeout", cli.unsupportedDiscoveryOption().?); + try std.testing.expectEqual(@as(?[]const u8, null), cli.unsupportedDiscoveryOption()); cli.options.timeout = null; cli.options.parallel = true; diff --git a/src/lib.zig b/src/lib.zig index a65b239..195ff36 100644 --- a/src/lib.zig +++ b/src/lib.zig @@ -5,6 +5,7 @@ pub const assertions = @import("assertions.zig"); pub const suite = @import("suite.zig"); pub const test_runner = @import("test_runner.zig"); pub const reporter = @import("reporter.zig"); +pub const pipeline = @import("pipeline.zig"); pub const matchers = @import("matchers.zig"); pub const mock = @import("mock.zig"); pub const cli = @import("cli.zig"); @@ -79,6 +80,15 @@ pub const JsonReporter = reporter.JsonReporter; pub const TAPReporter = reporter.TAPReporter; pub const JUnitReporter = reporter.JUnitReporter; pub const Colors = reporter.Colors; +pub const ReporterSet = reporter.ReporterSet; + +// Unified execution pipeline +pub const TestPlan = pipeline.TestPlan; +pub const PlanItem = pipeline.PlanItem; +pub const PlanOrigin = pipeline.PlanOrigin; +pub const ExecutionPolicy = pipeline.ExecutionPolicy; +pub const LifecycleEvent = pipeline.LifecycleEvent; +pub const EventStream = pipeline.EventStream; // Matchers pub const Matchers = matchers.Matchers; diff --git a/src/main.zig b/src/main.zig index 9c45360..4b59a7e 100644 --- a/src/main.zig +++ b/src/main.zig @@ -161,6 +161,11 @@ pub fn main(init: std.process.Init.Minimal) !void { } var all_passed: bool = undefined; + const stdout_file = std.Io.File.stdout(); + var stdout_buffer: [4096]u8 = undefined; + var reporter_io: std.Io.Threaded = .init(std.mem.Allocator.failing, .{ .environ = .empty }); + defer reporter_io.deinit(); + var stdout_writer = stdout_file.writer(reporter_io.io(), &stdout_buffer); const shard_options: ?lib.ShardOptions = if (cli_parser.options.shard_index) |index| .{ .index = index, .count = cli_parser.options.shard_count.? } else @@ -206,6 +211,10 @@ pub fn main(init: std.process.Init.Minimal) !void { .verbose = cli_parser.options.verbose, .use_colors = !cli_parser.options.no_color, .shard = shard_options, + .reporter_type = cli_parser.options.reporter, + .junit_output = cli_parser.options.junit_output orelse "test-results.xml", + .timeout_ms = cli_parser.options.timeout, + .reporter_writer = &stdout_writer.interface, .coverage_options = cov_opts, .ui_server = if (ui_server) |*server| server else null, }; @@ -245,6 +254,10 @@ pub fn main(init: std.process.Init.Minimal) !void { .verbose = cli_parser.options.verbose, .use_colors = !cli_parser.options.no_color, .shard = shard_options, + .reporter_type = cli_parser.options.reporter, + .junit_output = cli_parser.options.junit_output orelse "test-results.xml", + .timeout_ms = cli_parser.options.timeout, + .reporter_writer = &stdout_writer.interface, .coverage_options = cov_opts, .ui_server = if (ui_server) |*server| server else null, }; @@ -256,7 +269,8 @@ pub fn main(init: std.process.Init.Minimal) !void { defer lib.cleanupRegistry(); // Create test runner with CLI options - const runner_options = cli_parser.toRunnerOptions(); + var runner_options = cli_parser.toRunnerOptions(); + runner_options.reporter_writer = &stdout_writer.interface; var runner = lib.TestRunner.init(allocator, registry, runner_options); defer runner.deinit(); diff --git a/src/parallel.zig b/src/parallel.zig index 605c8c4..380d6a4 100644 --- a/src/parallel.zig +++ b/src/parallel.zig @@ -1,6 +1,7 @@ const std = @import("std"); const suite = @import("suite.zig"); const reporter = @import("reporter.zig"); +const pipeline = @import("pipeline.zig"); const compat = @import("compat.zig"); /// Options for bounded parallel test execution. @@ -10,6 +11,7 @@ pub const ParallelOptions = struct { n_jobs: ?usize = null, enabled: bool = false, filter: ?[]const u8 = null, + timeout_ms: ?u64 = null, }; pub const ParallelError = error{ @@ -31,6 +33,7 @@ const WorkerContext = struct { tests: []const *suite.TestCase, next_index: std.atomic.Value(usize) = std.atomic.Value(usize).init(0), error_mutex: compat.Mutex = .{}, + policy: pipeline.ExecutionPolicy, fn worker(self: *WorkerContext) void { while (true) { @@ -90,8 +93,15 @@ const WorkerContext = struct { test_case.error_message = std.fmt.allocPrint(self.allocator, format, args) catch "Out of memory"; } - fn finishTiming(_: *WorkerContext, test_case: *suite.TestCase, start_time: i128) void { + fn finishTiming(self: *WorkerContext, test_case: *suite.TestCase, start_time: i128) void { test_case.execution_time_ns = @intCast(compat.nanoTimestamp() - start_time); + if (test_case.status == .passed and self.policy.timedOut(test_case.execution_time_ns)) { + self.fail( + test_case, + "Test exceeded timeout of {d}ms", + .{self.policy.timeout_ms.?}, + ); + } } }; @@ -167,6 +177,10 @@ const RunContext = struct { .allocator = self.allocator, .test_suite = test_suite, .tests = tests, + .policy = .{ + .filter = self.options.filter, + .timeout_ms = self.options.timeout_ms, + }, }; const count = @min(self.worker_limit, tests.len); var threads: std.ArrayList(std.Thread) = .empty; diff --git a/src/pipeline.zig b/src/pipeline.zig new file mode 100644 index 0000000..7177b98 --- /dev/null +++ b/src/pipeline.zig @@ -0,0 +1,169 @@ +const std = @import("std"); +const discovery = @import("discovery.zig"); +const reporter = @import("reporter.zig"); +const suite = @import("suite.zig"); + +pub const PlanOrigin = enum { + registered, + discovered, +}; + +pub const RegisteredTest = struct { + test_suite: *suite.TestSuite, + test_case: *suite.TestCase, +}; + +pub const PlanItem = union(PlanOrigin) { + registered: RegisteredTest, + discovered: *const discovery.TestFile, + + pub fn identity(self: PlanItem) []const u8 { + return switch (self) { + .registered => |item| item.test_case.name, + .discovered => |file| file.relative_path, + }; + } +}; + +/// Executor-neutral plan produced by registration or file discovery. +pub const TestPlan = struct { + allocator: std.mem.Allocator, + items: std.ArrayList(PlanItem) = .empty, + + pub fn init(allocator: std.mem.Allocator) TestPlan { + return .{ .allocator = allocator }; + } + + pub fn deinit(self: *TestPlan) void { + self.items.deinit(self.allocator); + } + + pub fn fromRegistry( + allocator: std.mem.Allocator, + registry: *suite.TestRegistry, + ) !TestPlan { + var plan = TestPlan.init(allocator); + errdefer plan.deinit(); + for (registry.root_suites.items) |test_suite| { + try plan.appendSuite(test_suite); + } + return plan; + } + + pub fn fromDiscovery( + allocator: std.mem.Allocator, + discovered: *const discovery.DiscoveryResult, + ) !TestPlan { + var plan = TestPlan.init(allocator); + errdefer plan.deinit(); + for (discovered.files.items) |*file| { + try plan.appendDiscovered(file); + } + return plan; + } + + pub fn appendDiscovered(self: *TestPlan, file: *const discovery.TestFile) !void { + try self.items.append(self.allocator, .{ .discovered = file }); + } + + fn appendSuite(self: *TestPlan, test_suite: *suite.TestSuite) !void { + for (test_suite.tests.items) |*test_case| { + try self.items.append(self.allocator, .{ .registered = .{ + .test_suite = test_suite, + .test_case = test_case, + } }); + } + for (test_suite.suites.items) |nested| { + try self.appendSuite(nested); + } + } +}; + +/// Shared execution policy. Executors apply it at their natural granularity: +/// registered test names or filters forwarded to an external Zig test process. +pub const ExecutionPolicy = struct { + bail: bool = false, + filter: ?[]const u8 = null, + timeout_ms: ?u64 = null, + + pub fn matches(self: ExecutionPolicy, name: []const u8) bool { + const filter = self.filter orelse return true; + return std.mem.indexOf(u8, name, filter) != null; + } + + pub fn shouldStop(self: ExecutionPolicy, results: *const reporter.TestResults) bool { + return self.bail and results.failed > 0; + } + + pub fn timedOut(self: ExecutionPolicy, execution_time_ns: u64) bool { + const timeout_ms = self.timeout_ms orelse return false; + return execution_time_ns > timeout_ms * std.time.ns_per_ms; + } +}; + +pub const LifecycleEvent = union(enum) { + run_started: usize, + run_finished: *reporter.TestResults, + suite_started: []const u8, + suite_finished: []const u8, + test_started: []const u8, + test_finished: *const suite.TestCase, +}; + +/// Routes the typed lifecycle shared by every executor into existing reporter +/// callbacks. Additional consumers can be added without changing executors. +pub const EventStream = struct { + rep: *reporter.Reporter, + + pub fn emit(self: EventStream, event: LifecycleEvent) !void { + switch (event) { + .run_started => |total| try self.rep.onRunStart(total), + .run_finished => |results| try self.rep.onRunEnd(results), + .suite_started => |name| try self.rep.onSuiteStart(name), + .suite_finished => |name| try self.rep.onSuiteEnd(name), + .test_started => |name| try self.rep.onTestStart(name), + .test_finished => |test_case| try self.rep.onTestEnd(test_case), + } + } +}; + +fn passingTest(_: std.mem.Allocator) !void {} + +test "registered and discovered inputs produce one plan model" { + const allocator = std.testing.allocator; + + var registry = suite.TestRegistry.init(allocator); + defer registry.deinit(); + const test_suite = try suite.TestSuite.init(allocator, "unit"); + try test_suite.addTest(suite.TestCase.init("passes", passingTest)); + try registry.registerSuite(test_suite); + + var registered = try TestPlan.fromRegistry(allocator, ®istry); + defer registered.deinit(); + try std.testing.expectEqual(@as(usize, 1), registered.items.items.len); + try std.testing.expectEqualStrings("passes", registered.items.items[0].identity()); + + var discovered = discovery.DiscoveryResult.init(allocator); + defer discovered.deinit(); + try discovered.addFile("tests/sample.test.zig", "sample.test.zig", "sample.test.zig"); + + var external = try TestPlan.fromDiscovery(allocator, &discovered); + defer external.deinit(); + try std.testing.expectEqual(@as(usize, 1), external.items.items.len); + try std.testing.expectEqualStrings("sample.test.zig", external.items.items[0].identity()); +} + +test "execution policy shares matching and bail semantics" { + const policy = ExecutionPolicy{ .bail = true, .filter = "selected", .timeout_ms = 500 }; + try std.testing.expect(policy.matches("selected test")); + try std.testing.expect(!policy.matches("other test")); + try std.testing.expectEqual(@as(?u64, 500), policy.timeout_ms); + try std.testing.expect(!policy.timedOut(499 * std.time.ns_per_ms)); + try std.testing.expect(policy.timedOut(501 * std.time.ns_per_ms)); + + var results = reporter.TestResults.init(std.testing.allocator); + defer results.deinit(); + try std.testing.expect(!policy.shouldStop(&results)); + results.failed = 1; + try std.testing.expect(policy.shouldStop(&results)); +} diff --git a/src/reporter.zig b/src/reporter.zig index 2efb915..0d2148a 100644 --- a/src/reporter.zig +++ b/src/reporter.zig @@ -2,6 +2,14 @@ const std = @import("std"); const suite = @import("suite.zig"); const compat = @import("compat.zig"); +pub const ReporterType = enum { + spec, + dot, + json, + tap, + junit, +}; + /// ANSI color codes pub const Colors = struct { pub const reset = "\x1b[0m"; @@ -101,6 +109,7 @@ pub const SpecReporter = struct { reporter: Reporter, indent_level: usize = 0, writer: std.Io.Writer, + writer_ref: ?*std.Io.Writer = null, const Self = @This(); @@ -114,6 +123,16 @@ pub const SpecReporter = struct { }; } + pub fn initRef(allocator: std.mem.Allocator, writer: *std.Io.Writer) Self { + var result = init(allocator, writer.*); + result.writer_ref = writer; + return result; + } + + fn output(instance: *Self) *std.Io.Writer { + return instance.writer_ref orelse &instance.writer; + } + const vtable = Reporter.VTable{ .onRunStart = onRunStart, .onRunEnd = onRunEnd, @@ -129,64 +148,64 @@ pub const SpecReporter = struct { fn onRunStart(reporter: *Reporter, total_tests: usize) !void { const s = self(reporter); - try s.writer.print("\n", .{}); + try s.output().print("\n", .{}); if (reporter.use_colors) { - try s.writer.print("{s}Running {d} test(s)...{s}\n\n", .{ Colors.bold, total_tests, Colors.reset }); + try s.output().print("{s}Running {d} test(s)...{s}\n\n", .{ Colors.bold, total_tests, Colors.reset }); } else { - try s.writer.print("Running {d} test(s)...\n\n", .{total_tests}); + try s.output().print("Running {d} test(s)...\n\n", .{total_tests}); } } fn onRunEnd(reporter: *Reporter, results: *TestResults) !void { const s = self(reporter); - try s.writer.print("\n", .{}); + try s.output().print("\n", .{}); // Print failed tests details if (results.failed > 0) { if (reporter.use_colors) { - try s.writer.print("{s}Failed Tests:{s}\n\n", .{ Colors.bold ++ Colors.red, Colors.reset }); + try s.output().print("{s}Failed Tests:{s}\n\n", .{ Colors.bold ++ Colors.red, Colors.reset }); } else { - try s.writer.print("Failed Tests:\n\n", .{}); + try s.output().print("Failed Tests:\n\n", .{}); } for (results.failed_tests.items) |test_case| { if (reporter.use_colors) { - try s.writer.print(" {s}✗{s} {s}\n", .{ Colors.red, Colors.reset, test_case.name }); + try s.output().print(" {s}✗{s} {s}\n", .{ Colors.red, Colors.reset, test_case.name }); } else { - try s.writer.print(" ✗ {s}\n", .{test_case.name}); + try s.output().print(" ✗ {s}\n", .{test_case.name}); } if (test_case.error_message) |msg| { - try s.writer.print(" {s}\n", .{msg}); + try s.output().print(" {s}\n", .{msg}); } } - try s.writer.print("\n", .{}); + try s.output().print("\n", .{}); } // Print summary const total_time_ms = @as(f64, @floatFromInt(results.total_time_ns)) / 1_000_000.0; if (reporter.use_colors) { - try s.writer.print("{s}Test Summary:{s}\n", .{ Colors.bold, Colors.reset }); - try s.writer.print(" Total: {d}\n", .{results.total}); - try s.writer.print(" {s}Passed: {d}{s}\n", .{ Colors.green, results.passed, Colors.reset }); + try s.output().print("{s}Test Summary:{s}\n", .{ Colors.bold, Colors.reset }); + try s.output().print(" Total: {d}\n", .{results.total}); + try s.output().print(" {s}Passed: {d}{s}\n", .{ Colors.green, results.passed, Colors.reset }); if (results.failed > 0) { - try s.writer.print(" {s}Failed: {d}{s}\n", .{ Colors.red, results.failed, Colors.reset }); + try s.output().print(" {s}Failed: {d}{s}\n", .{ Colors.red, results.failed, Colors.reset }); } if (results.skipped > 0) { - try s.writer.print(" {s}Skipped: {d}{s}\n", .{ Colors.yellow, results.skipped, Colors.reset }); + try s.output().print(" {s}Skipped: {d}{s}\n", .{ Colors.yellow, results.skipped, Colors.reset }); } - try s.writer.print(" Time: {d:.2}ms\n", .{total_time_ms}); + try s.output().print(" Time: {d:.2}ms\n", .{total_time_ms}); } else { - try s.writer.print("Test Summary:\n", .{}); - try s.writer.print(" Total: {d}\n", .{results.total}); - try s.writer.print(" Passed: {d}\n", .{results.passed}); + try s.output().print("Test Summary:\n", .{}); + try s.output().print(" Total: {d}\n", .{results.total}); + try s.output().print(" Passed: {d}\n", .{results.passed}); if (results.failed > 0) { - try s.writer.print(" Failed: {d}\n", .{results.failed}); + try s.output().print(" Failed: {d}\n", .{results.failed}); } if (results.skipped > 0) { - try s.writer.print(" Skipped: {d}\n", .{results.skipped}); + try s.output().print(" Skipped: {d}\n", .{results.skipped}); } - try s.writer.print(" Time: {d:.2}ms\n", .{total_time_ms}); + try s.output().print(" Time: {d:.2}ms\n", .{total_time_ms}); } } @@ -196,13 +215,13 @@ pub const SpecReporter = struct { // Print indent var i: usize = 0; while (i < s.indent_level) : (i += 1) { - try s.writer.print(" ", .{}); + try s.output().print(" ", .{}); } if (reporter.use_colors) { - try s.writer.print("{s}{s}{s}\n", .{ Colors.bold, suite_name, Colors.reset }); + try s.output().print("{s}{s}{s}\n", .{ Colors.bold, suite_name, Colors.reset }); } else { - try s.writer.print("{s}\n", .{suite_name}); + try s.output().print("{s}\n", .{suite_name}); } s.indent_level += 1; } @@ -226,7 +245,7 @@ pub const SpecReporter = struct { // Print indent var i: usize = 0; while (i < s.indent_level) : (i += 1) { - try s.writer.print(" ", .{}); + try s.output().print(" ", .{}); } const time_ms = @as(f64, @floatFromInt(test_case.execution_time_ns)) / 1_000_000.0; @@ -234,7 +253,7 @@ pub const SpecReporter = struct { switch (test_case.status) { .passed => { if (reporter.use_colors) { - try s.writer.print("{s}✓{s} {s} {s}({d:.2}ms){s}\n", .{ + try s.output().print("{s}✓{s} {s} {s}({d:.2}ms){s}\n", .{ Colors.green, Colors.reset, test_case.name, @@ -243,12 +262,12 @@ pub const SpecReporter = struct { Colors.reset, }); } else { - try s.writer.print("✓ {s} ({d:.2}ms)\n", .{ test_case.name, time_ms }); + try s.output().print("✓ {s} ({d:.2}ms)\n", .{ test_case.name, time_ms }); } }, .failed => { if (reporter.use_colors) { - try s.writer.print("{s}✗{s} {s} {s}({d:.2}ms){s}\n", .{ + try s.output().print("{s}✗{s} {s} {s}({d:.2}ms){s}\n", .{ Colors.red, Colors.reset, test_case.name, @@ -257,12 +276,12 @@ pub const SpecReporter = struct { Colors.reset, }); } else { - try s.writer.print("✗ {s} ({d:.2}ms)\n", .{ test_case.name, time_ms }); + try s.output().print("✗ {s} ({d:.2}ms)\n", .{ test_case.name, time_ms }); } }, .skipped => { if (reporter.use_colors) { - try s.writer.print("{s}⊘{s} {s} {s}(skipped){s}\n", .{ + try s.output().print("{s}⊘{s} {s} {s}(skipped){s}\n", .{ Colors.yellow, Colors.reset, test_case.name, @@ -270,7 +289,7 @@ pub const SpecReporter = struct { Colors.reset, }); } else { - try s.writer.print("⊘ {s} (skipped)\n", .{test_case.name}); + try s.output().print("⊘ {s} (skipped)\n", .{test_case.name}); } }, else => {}, @@ -282,6 +301,7 @@ pub const SpecReporter = struct { pub const DotReporter = struct { reporter: Reporter, writer: std.Io.Writer, + writer_ref: ?*std.Io.Writer = null, tests_per_line: usize = 80, current_line_count: usize = 0, @@ -297,6 +317,16 @@ pub const DotReporter = struct { }; } + pub fn initRef(allocator: std.mem.Allocator, writer: *std.Io.Writer) Self { + var result = init(allocator, writer.*); + result.writer_ref = writer; + return result; + } + + fn output(instance: *Self) *std.Io.Writer { + return instance.writer_ref orelse &instance.writer; + } + const vtable = Reporter.VTable{ .onRunStart = onRunStart, .onRunEnd = onRunEnd, @@ -312,21 +342,21 @@ pub const DotReporter = struct { fn onRunStart(reporter: *Reporter, total_tests: usize) !void { const s = self(reporter); - try s.writer.print("\nRunning {d} tests:\n", .{total_tests}); + try s.output().print("\nRunning {d} tests:\n", .{total_tests}); } fn onRunEnd(reporter: *Reporter, results: *TestResults) !void { const s = self(reporter); - try s.writer.print("\n\n", .{}); + try s.output().print("\n\n", .{}); const total_time_ms = @as(f64, @floatFromInt(results.total_time_ns)) / 1_000_000.0; if (reporter.use_colors) { - try s.writer.print("{s}Passed: {d}{s}, ", .{ Colors.green, results.passed, Colors.reset }); - try s.writer.print("{s}Failed: {d}{s}, ", .{ Colors.red, results.failed, Colors.reset }); - try s.writer.print("Total: {d} ({d:.2}ms)\n", .{ results.total, total_time_ms }); + try s.output().print("{s}Passed: {d}{s}, ", .{ Colors.green, results.passed, Colors.reset }); + try s.output().print("{s}Failed: {d}{s}, ", .{ Colors.red, results.failed, Colors.reset }); + try s.output().print("Total: {d} ({d:.2}ms)\n", .{ results.total, total_time_ms }); } else { - try s.writer.print("Passed: {d}, Failed: {d}, Total: {d} ({d:.2}ms)\n", .{ + try s.output().print("Passed: {d}, Failed: {d}, Total: {d} ({d:.2}ms)\n", .{ results.passed, results.failed, results.total, @@ -356,23 +386,23 @@ pub const DotReporter = struct { switch (test_case.status) { .passed => { if (reporter.use_colors) { - try s.writer.print("{s}.{s}", .{ Colors.green, Colors.reset }); + try s.output().print("{s}.{s}", .{ Colors.green, Colors.reset }); } else { - try s.writer.print(".", .{}); + try s.output().print(".", .{}); } }, .failed => { if (reporter.use_colors) { - try s.writer.print("{s}F{s}", .{ Colors.red, Colors.reset }); + try s.output().print("{s}F{s}", .{ Colors.red, Colors.reset }); } else { - try s.writer.print("F", .{}); + try s.output().print("F", .{}); } }, .skipped => { if (reporter.use_colors) { - try s.writer.print("{s}S{s}", .{ Colors.yellow, Colors.reset }); + try s.output().print("{s}S{s}", .{ Colors.yellow, Colors.reset }); } else { - try s.writer.print("S", .{}); + try s.output().print("S", .{}); } }, else => {}, @@ -380,7 +410,7 @@ pub const DotReporter = struct { s.current_line_count += 1; if (s.current_line_count >= s.tests_per_line) { - try s.writer.print("\n", .{}); + try s.output().print("\n", .{}); s.current_line_count = 0; } } @@ -390,6 +420,7 @@ pub const DotReporter = struct { pub const JsonReporter = struct { reporter: Reporter, writer: std.Io.Writer, + writer_ref: ?*std.Io.Writer = null, suites: std.ArrayList([]const u8), const Self = @This(); @@ -405,6 +436,16 @@ pub const JsonReporter = struct { }; } + pub fn initRef(allocator: std.mem.Allocator, writer: *std.Io.Writer) Self { + var result = init(allocator, writer.*); + result.writer_ref = writer; + return result; + } + + fn output(instance: *Self) *std.Io.Writer { + return instance.writer_ref orelse &instance.writer; + } + pub fn deinit(s: *Self) void { s.suites.deinit(s.reporter.allocator); } @@ -424,14 +465,14 @@ pub const JsonReporter = struct { fn onRunStart(reporter: *Reporter, total_tests: usize) !void { const s = self(reporter); - try s.writer.print("{{\"totalTests\":{d},\"tests\":[\n", .{total_tests}); + try s.output().print("{{\"totalTests\":{d},\"tests\":[\n", .{total_tests}); } fn onRunEnd(reporter: *Reporter, results: *TestResults) !void { const s = self(reporter); const total_time_ms = @as(f64, @floatFromInt(results.total_time_ns)) / 1_000_000.0; - try s.writer.print("],\"summary\":{{\"total\":{d},\"passed\":{d},\"failed\":{d},\"skipped\":{d},\"time\":{d:.2}}}}}\n", .{ + try s.output().print("],\"summary\":{{\"total\":{d},\"passed\":{d},\"failed\":{d},\"skipped\":{d},\"time\":{d:.2}}}}}\n", .{ results.total, results.passed, results.failed, @@ -470,17 +511,17 @@ pub const JsonReporter = struct { }; // Note: In a real implementation, you'd want to properly escape JSON strings - try s.writer.print(" {{\"name\":\"{s}\",\"status\":\"{s}\",\"time\":{d:.2}", .{ + try s.output().print(" {{\"name\":\"{s}\",\"status\":\"{s}\",\"time\":{d:.2}", .{ test_case.name, status_str, time_ms, }); if (test_case.error_message) |msg| { - try s.writer.print(",\"error\":\"{s}\"", .{msg}); + try s.output().print(",\"error\":\"{s}\"", .{msg}); } - try s.writer.print("}},\n", .{}); + try s.output().print("}},\n", .{}); } }; @@ -488,6 +529,7 @@ pub const JsonReporter = struct { pub const TAPReporter = struct { reporter: Reporter, writer: std.Io.Writer, + writer_ref: ?*std.Io.Writer = null, test_count: usize = 0, const Self = @This(); @@ -510,9 +552,19 @@ pub const TAPReporter = struct { }; } + pub fn initRef(allocator: std.mem.Allocator, writer: *std.Io.Writer) Self { + var result = init(allocator, writer.*); + result.writer_ref = writer; + return result; + } + + fn output(instance: *Self) *std.Io.Writer { + return instance.writer_ref orelse &instance.writer; + } + fn onRunStart(reporter: *Reporter, total: usize) !void { const self: *Self = @fieldParentPtr("reporter", reporter); - try self.writer.print("TAP version 14\n1..{d}\n", .{total}); + try self.output().print("TAP version 14\n1..{d}\n", .{total}); } fn onRunEnd(reporter: *Reporter, results: *TestResults) !void { @@ -522,7 +574,7 @@ pub const TAPReporter = struct { fn onSuiteStart(reporter: *Reporter, suite_name: []const u8) !void { const self: *Self = @fieldParentPtr("reporter", reporter); - try self.writer.print("# Subtest: {s}\n", .{suite_name}); + try self.output().print("# Subtest: {s}\n", .{suite_name}); } fn onSuiteEnd(reporter: *Reporter, suite_name: []const u8) !void { @@ -546,14 +598,14 @@ pub const TAPReporter = struct { else => "not ok", }; - try self.writer.print("{s} {d} - {s}", .{ status, self.test_count, test_case.name }); + try self.output().print("{s} {d} - {s}", .{ status, self.test_count, test_case.name }); if (test_case.status == .skipped) { - try self.writer.print(" # SKIP\n", .{}); + try self.output().print(" # SKIP\n", .{}); } else if (test_case.status == .failed and test_case.error_message != null) { - try self.writer.print("\n ---\n message: {s}\n ...\n", .{test_case.error_message.?}); + try self.output().print("\n ---\n message: {s}\n ...\n", .{test_case.error_message.?}); } else { - try self.writer.print("\n", .{}); + try self.output().print("\n", .{}); } } }; @@ -735,6 +787,80 @@ pub const JUnitReporter = struct { } }; +/// Owns every built-in reporter and selects one through a shared interface. +/// Keeping construction here lets every executor expose identical reporters. +pub const ReporterSet = struct { + kind: ReporterType, + spec: SpecReporter, + dot: DotReporter, + json: JsonReporter, + tap: TAPReporter, + junit: JUnitReporter, + + pub fn init( + allocator: std.mem.Allocator, + writer: std.Io.Writer, + kind: ReporterType, + junit_output: []const u8, + use_colors: bool, + ) ReporterSet { + var reporters = ReporterSet{ + .kind = kind, + .spec = SpecReporter.init(allocator, writer), + .dot = DotReporter.init(allocator, writer), + .json = JsonReporter.init(allocator, writer), + .tap = TAPReporter.init(allocator, writer), + .junit = JUnitReporter.init(allocator, junit_output), + }; + reporters.selected().use_colors = use_colors; + return reporters; + } + + pub fn initRef( + allocator: std.mem.Allocator, + writer: *std.Io.Writer, + kind: ReporterType, + junit_output: []const u8, + use_colors: bool, + ) ReporterSet { + var reporters = ReporterSet{ + .kind = kind, + .spec = SpecReporter.initRef(allocator, writer), + .dot = DotReporter.initRef(allocator, writer), + .json = JsonReporter.initRef(allocator, writer), + .tap = TAPReporter.initRef(allocator, writer), + .junit = JUnitReporter.init(allocator, junit_output), + }; + reporters.selected().use_colors = use_colors; + return reporters; + } + + pub fn deinit(self: *ReporterSet) void { + self.json.deinit(); + self.junit.deinit(); + } + + pub fn selected(self: *ReporterSet) *Reporter { + return switch (self.kind) { + .spec => &self.spec.reporter, + .dot => &self.dot.reporter, + .json => &self.json.reporter, + .tap => &self.tap.reporter, + .junit => &self.junit.reporter, + }; + } + + pub fn flush(self: *ReporterSet) !void { + switch (self.kind) { + .spec => try self.spec.output().flush(), + .dot => try self.dot.output().flush(), + .json => try self.json.output().flush(), + .tap => try self.tap.output().flush(), + .junit => {}, + } + } +}; + // Tests test "Colors constants exist" { // Just verify the constants are defined diff --git a/src/test_loader.zig b/src/test_loader.zig index 256b799..be5943f 100644 --- a/src/test_loader.zig +++ b/src/test_loader.zig @@ -1,6 +1,9 @@ const std = @import("std"); const discovery = @import("discovery.zig"); const sharding = @import("sharding.zig"); +const pipeline = @import("pipeline.zig"); +const reporter = @import("reporter.zig"); +const suite = @import("suite.zig"); const coverage = @import("coverage.zig"); const ui_server = @import("ui_server.zig"); const compat = @import("compat.zig"); @@ -17,6 +20,15 @@ pub const LoaderOptions = struct { use_colors: bool = true, /// Optional one-based file shard to execute shard: ?sharding.ShardOptions = null, + /// Reporter shared with programmatic execution + reporter_type: reporter.ReporterType = .spec, + /// Output path used by the JUnit reporter + junit_output: []const u8 = "test-results.xml", + /// Global per-file timeout checked by the shared execution policy + timeout_ms: ?u64 = null, + /// Optional output supplied by a CLI host. Embedded/test callers default + /// to stderr so they do not interfere with Zig's stdout test protocol. + reporter_writer: ?*std.Io.Writer = null, /// Coverage options coverage_options: ?coverage.CoverageOptions = null, /// UI server for real-time updates @@ -34,7 +46,9 @@ pub fn runDiscoveredTests( return false; } - const selected_files = try selectedFileCount(discovered, options.shard); + var plan = try discoveryPlan(allocator, discovered, options.shard); + defer plan.deinit(); + const selected_files = plan.items.items.len; if (options.shard) |shard| { std.debug.print( "Shard {}/{}: selected {} of {} test file(s).\n", @@ -43,12 +57,37 @@ pub fn runDiscoveredTests( } std.debug.print("Found {} test file(s):\n", .{selected_files}); - for (discovered.files.items) |file| { - if (!try fileSelected(file.relative_path, options.shard)) continue; + for (plan.items.items) |item| { + const file = item.discovered; std.debug.print(" - {s}\n", .{file.relative_path}); } std.debug.print("\n", .{}); + const stderr_file = std.Io.File.stderr(); + var stderr_buffer: [4096]u8 = undefined; + var threaded_io: std.Io.Threaded = .init(std.mem.Allocator.failing, .{ .environ = .empty }); + defer threaded_io.deinit(); + var stderr_writer = stderr_file.writer(threaded_io.io(), &stderr_buffer); + const reporter_writer = options.reporter_writer orelse &stderr_writer.interface; + var reporters = reporter.ReporterSet.initRef( + allocator, + reporter_writer, + options.reporter_type, + options.junit_output, + options.use_colors, + ); + defer reporters.deinit(); + const events = pipeline.EventStream{ .rep = reporters.selected() }; + const policy = pipeline.ExecutionPolicy{ + .bail = options.bail, + .filter = options.filter, + .timeout_ms = options.timeout_ms, + }; + var results = reporter.TestResults.init(allocator); + defer results.deinit(); + + try events.emit(.{ .run_started = selected_files }); + // Notify UI of run start if (options.ui_server) |server| { var buffer: [256]u8 = undefined; @@ -56,8 +95,6 @@ pub fn runDiscoveredTests( try server.broadcast("run_start", json); } - var total_passed: usize = 0; - var total_failed: usize = 0; var files_run: usize = 0; // Clean coverage directory once before all tests if coverage is enabled @@ -68,11 +105,12 @@ pub fn runDiscoveredTests( } } - for (discovered.files.items) |file| { - if (!try fileSelected(file.relative_path, options.shard)) continue; - + for (plan.items.items) |item| { + const file = item.discovered; // Run each test file using zig test command std.debug.print("Running {s}...\n", .{file.relative_path}); + try events.emit(.{ .suite_started = file.relative_path }); + try events.emit(.{ .test_started = file.name }); // Notify UI of test file start if (options.ui_server) |server| { @@ -83,11 +121,25 @@ pub fn runDiscoveredTests( } const start_time = compat.nanoTimestamp(); - const result = try runTestFile(allocator, file.path, options); + var result = try runTestFile(allocator, file.path, options); const end_time = compat.nanoTimestamp(); const execution_time_ns: u64 = @intCast(end_time - start_time); + const timed_out = policy.timedOut(execution_time_ns); + if (timed_out) result = false; files_run += 1; + var test_case = suite.TestCase.init(file.name, externalTestNoop); + test_case.file = file.relative_path; + test_case.execution_time_ns = execution_time_ns; + test_case.status = if (result) .passed else .failed; + if (timed_out) { + test_case.error_message = "External test file exceeded the configured timeout"; + } else if (!result) { + test_case.error_message = "External zig test process failed"; + } + try results.addTest(&test_case); + try events.emit(.{ .test_finished = &test_case }); + try events.emit(.{ .suite_finished = file.relative_path }); // Notify UI of test file end if (options.ui_server) |server| { @@ -99,15 +151,10 @@ pub fn runDiscoveredTests( } if (result) { - total_passed += 1; - if (options.verbose) { - std.debug.print(" ✓ {s} passed\n", .{file.name}); - } + if (options.verbose) std.debug.print(" ✓ {s} passed\n", .{file.name}); } else { - total_failed += 1; std.debug.print(" ✗ {s} failed\n", .{file.name}); - - if (options.bail) { + if (policy.shouldStop(&results)) { std.debug.print("\nStopping on first failure (--bail)\n", .{}); break; } @@ -118,48 +165,49 @@ pub fn runDiscoveredTests( if (options.coverage_options) |cov_opts| { if (cov_opts.enabled and files_run > 0) { std.debug.print("\n", .{}); - const cov_result = coverage.parseCoverageReport(allocator, cov_opts.output_dir) catch |err| { + const cov_result: ?coverage.CoverageResult = coverage.parseCoverageReport(allocator, cov_opts.output_dir) catch |err| result: { std.debug.print("Warning: Could not parse coverage report: {any}\n", .{err}); - return total_failed == 0; + break :result null; }; - - coverage.printCoverageSummary(cov_result); + if (cov_result) |summary| coverage.printCoverageSummary(summary); } } // Notify UI of run end if (options.ui_server) |server| { var buffer: [512]u8 = undefined; - const json = try std.fmt.bufPrint(&buffer, "{{\"total\":{d},\"passed\":{d},\"failed\":{d},\"skipped\":0}}", .{ files_run, total_passed, total_failed }); + const json = try std.fmt.bufPrint(&buffer, "{{\"total\":{d},\"passed\":{d},\"failed\":{d},\"skipped\":{d}}}", .{ results.total, results.passed, results.failed, results.skipped }); try server.broadcast("run_end", json); } - std.debug.print("\n", .{}); - std.debug.print("Test Summary:\n", .{}); if (options.shard) |shard| { + std.debug.print("\nShard Summary:\n", .{}); std.debug.print(" Shard: {}/{}\n", .{ shard.index, shard.count }); std.debug.print(" Selected: {} of {} files\n", .{ selected_files, discovered.files.items.len }); } - std.debug.print(" Files run: {}\n", .{files_run}); - std.debug.print(" Passed: {}\n", .{total_passed}); - std.debug.print(" Failed: {}\n", .{total_failed}); + try events.emit(.{ .run_finished = &results }); + try reporters.flush(); - return total_failed == 0; + return results.failed == 0; } +fn externalTestNoop(_: std.mem.Allocator) !void {} + fn fileSelected(relative_path: []const u8, shard: ?sharding.ShardOptions) !bool { return if (shard) |selection| selection.includes(relative_path) else true; } -fn selectedFileCount( - discovered: *const discovery.DiscoveryResult, +fn discoveryPlan( + allocator: std.mem.Allocator, + discovered: *discovery.DiscoveryResult, shard: ?sharding.ShardOptions, -) !usize { - var count: usize = 0; - for (discovered.files.items) |file| { - if (try fileSelected(file.relative_path, shard)) count += 1; +) !pipeline.TestPlan { + var plan = pipeline.TestPlan.init(allocator); + errdefer plan.deinit(); + for (discovered.files.items) |*file| { + if (try fileSelected(file.relative_path, shard)) try plan.appendDiscovered(file); } - return count; + return plan; } /// Run a single test file using `zig test` @@ -241,6 +289,10 @@ test "LoaderOptions default values" { try std.testing.expectEqual(false, options.verbose); try std.testing.expectEqual(true, options.use_colors); try std.testing.expectEqual(@as(?sharding.ShardOptions, null), options.shard); + try std.testing.expectEqual(reporter.ReporterType.spec, options.reporter_type); + try std.testing.expectEqualStrings("test-results.xml", options.junit_output); + try std.testing.expectEqual(@as(?u64, null), options.timeout_ms); + try std.testing.expectEqual(@as(?*std.Io.Writer, null), options.reporter_writer); try std.testing.expectEqual(@as(?coverage.CoverageOptions, null), options.coverage_options); } @@ -303,10 +355,9 @@ test "file sharding partitions discovered files exactly once" { var total_selected: usize = 0; for (1..4) |index| { - total_selected += try selectedFileCount( - &discovered, - .{ .index = index, .count = 3 }, - ); + var plan = try discoveryPlan(allocator, &discovered, .{ .index = index, .count = 3 }); + defer plan.deinit(); + total_selected += plan.items.items.len; } try std.testing.expectEqual(discovered.files.items.len, total_selected); diff --git a/src/test_runner.zig b/src/test_runner.zig index e32fdfa..5c57a86 100644 --- a/src/test_runner.zig +++ b/src/test_runner.zig @@ -1,6 +1,7 @@ const std = @import("std"); const suite = @import("suite.zig"); const reporter_mod = @import("reporter.zig"); +const pipeline = @import("pipeline.zig"); const parallel = @import("parallel.zig"); const compat = @import("compat.zig"); @@ -16,15 +17,14 @@ pub const RunnerOptions = struct { use_colors: bool = true, parallel: bool = false, // Enable parallel execution n_jobs: ?usize = null, // Number of parallel jobs + junit_output: []const u8 = "test-results.xml", + timeout_ms: ?u64 = null, + /// Optional output supplied by a CLI host. Embedded/test callers default + /// to stderr so they do not interfere with Zig's stdout test protocol. + reporter_writer: ?*std.Io.Writer = null, }; -pub const ReporterType = enum { - spec, - dot, - json, - tap, - junit, -}; +pub const ReporterType = reporter_mod.ReporterType; pub const TestRunner = struct { allocator: std.mem.Allocator, @@ -49,32 +49,27 @@ pub const TestRunner = struct { /// Run all registered tests pub fn run(self: *Self) !bool { - const stdout_file = std.Io.File.stdout(); - var stdout_buffer: [4096]u8 = undefined; + const stderr_file = std.Io.File.stderr(); + var stderr_buffer: [4096]u8 = undefined; var threaded_io: std.Io.Threaded = .init(std.mem.Allocator.failing, .{ .environ = .empty }); defer threaded_io.deinit(); - var stdout_writer = stdout_file.writer(threaded_io.io(), &stdout_buffer); - - // Create reporter - var spec_reporter = reporter_mod.SpecReporter.init(self.allocator, stdout_writer.interface); - var dot_reporter = reporter_mod.DotReporter.init(self.allocator, stdout_writer.interface); - var json_reporter = reporter_mod.JsonReporter.init(self.allocator, stdout_writer.interface); - defer json_reporter.deinit(); - var tap_reporter = reporter_mod.TAPReporter.init(self.allocator, stdout_writer.interface); - var junit_reporter = reporter_mod.JUnitReporter.init(self.allocator, "test-results.xml"); - defer junit_reporter.deinit(); - - var current_reporter: *reporter_mod.Reporter = switch (self.options.reporter_type) { - .spec => &spec_reporter.reporter, - .dot => &dot_reporter.reporter, - .json => &json_reporter.reporter, - .tap => &tap_reporter.reporter, - .junit => &junit_reporter.reporter, - }; - - current_reporter.use_colors = self.options.use_colors; - - const total_tests = self.registry.countAllTests(); + var stderr_writer = stderr_file.writer(threaded_io.io(), &stderr_buffer); + const reporter_writer = self.options.reporter_writer orelse &stderr_writer.interface; + + var reporters = reporter_mod.ReporterSet.initRef( + self.allocator, + reporter_writer, + self.options.reporter_type, + self.options.junit_output, + self.options.use_colors, + ); + defer reporters.deinit(); + const current_reporter = reporters.selected(); + const events = pipeline.EventStream{ .rep = current_reporter }; + + var plan = try pipeline.TestPlan.fromRegistry(self.allocator, self.registry); + defer plan.deinit(); + const total_tests = plan.items.items.len; if (total_tests == 0) { return RunnerError.NoTestsFound; } @@ -85,6 +80,7 @@ pub const TestRunner = struct { .enabled = true, .n_jobs = self.options.n_jobs, .filter = self.options.filter, + .timeout_ms = self.options.timeout_ms, }; const all_passed = try parallel.runTestsParallel( @@ -98,24 +94,24 @@ pub const TestRunner = struct { try self.addSuiteResults(test_suite); } - try stdout_writer.interface.flush(); + try reporters.flush(); return all_passed and self.results.failed == 0; } // Sequential execution (original behavior) - try current_reporter.onRunStart(total_tests); + try events.emit(.{ .run_started = total_tests }); for (self.registry.root_suites.items) |test_suite| { - try self.runSuite(test_suite, current_reporter); - if (self.options.bail and self.results.failed > 0) { + try self.runSuite(test_suite, events); + if (self.policy().shouldStop(&self.results)) { break; } } // Notify reporter of run end - try current_reporter.onRunEnd(&self.results); + try events.emit(.{ .run_finished = &self.results }); // Flush output - try stdout_writer.interface.flush(); + try reporters.flush(); return self.results.failed == 0; } @@ -126,26 +122,26 @@ pub const TestRunner = struct { } /// Run a single test suite - fn runSuite(self: *Self, test_suite: *suite.TestSuite, rep: *reporter_mod.Reporter) !void { + fn runSuite(self: *Self, test_suite: *suite.TestSuite, events: pipeline.EventStream) !void { // Skip if marked as skip or if has_only and this isn't marked as only if (test_suite.shouldSkip()) { - try self.skipAllTests(test_suite); + try self.skipAllTests(test_suite, events); return; } if (self.registry.has_only and !test_suite.hasOnly()) { - try self.skipAllTests(test_suite); + try self.skipAllTests(test_suite, events); return; } // Notify reporter - try rep.onSuiteStart(test_suite.name); + try events.emit(.{ .suite_started = test_suite.name }); // Run beforeAll hooks test_suite.runBeforeAllHooks(self.allocator) catch |err| { std.debug.print("beforeAll hook failed: {any}\n", .{err}); - try self.skipAllTests(test_suite); - try rep.onSuiteEnd(test_suite.name); + try self.skipAllTests(test_suite, events); + try events.emit(.{ .suite_finished = test_suite.name }); return; }; @@ -153,32 +149,30 @@ pub const TestRunner = struct { for (test_suite.tests.items) |*test_case| { if (test_case.skip or (self.registry.has_only and !test_case.only)) { test_case.status = .skipped; - try rep.onTestEnd(test_case); + try events.emit(.{ .test_finished = test_case }); try self.results.addTest(test_case); continue; } // Check filter - if (self.options.filter) |filter| { - if (std.mem.indexOf(u8, test_case.name, filter) == null) { - test_case.status = .skipped; - try rep.onTestEnd(test_case); - try self.results.addTest(test_case); - continue; - } + if (!self.policy().matches(test_case.name)) { + test_case.status = .skipped; + try events.emit(.{ .test_finished = test_case }); + try self.results.addTest(test_case); + continue; } - try self.runTest(test_case, test_suite, rep); + try self.runTest(test_case, test_suite, events); - if (self.options.bail and test_case.status == .failed) { + if (self.policy().shouldStop(&self.results)) { break; } } // Run nested suites for (test_suite.suites.items) |nested_suite| { - try self.runSuite(nested_suite, rep); - if (self.options.bail and self.results.failed > 0) { + try self.runSuite(nested_suite, events); + if (self.policy().shouldStop(&self.results)) { break; } } @@ -188,12 +182,12 @@ pub const TestRunner = struct { std.debug.print("afterAll hook failed: {any}\n", .{err}); }; - try rep.onSuiteEnd(test_suite.name); + try events.emit(.{ .suite_finished = test_suite.name }); } /// Run a single test - fn runTest(self: *Self, test_case: *suite.TestCase, test_suite: *suite.TestSuite, rep: *reporter_mod.Reporter) !void { - try rep.onTestStart(test_case.name); + fn runTest(self: *Self, test_case: *suite.TestCase, test_suite: *suite.TestSuite, events: pipeline.EventStream) !void { + try events.emit(.{ .test_started = test_case.name }); test_case.status = .running; const start_time = compat.nanoTimestamp(); @@ -240,22 +234,39 @@ pub const TestRunner = struct { const end_time = compat.nanoTimestamp(); test_case.execution_time_ns = @intCast(end_time - start_time); + if (test_case.status == .passed and self.policy().timedOut(test_case.execution_time_ns)) { + test_case.status = .failed; + test_case.error_message = try std.fmt.allocPrint( + self.allocator, + "Test exceeded timeout of {d}ms", + .{self.options.timeout_ms.?}, + ); + } try self.results.addTest(test_case); - try rep.onTestEnd(test_case); + try events.emit(.{ .test_finished = test_case }); } /// Skip all tests in a suite - fn skipAllTests(self: *Self, test_suite: *suite.TestSuite) !void { + fn skipAllTests(self: *Self, test_suite: *suite.TestSuite, events: pipeline.EventStream) !void { for (test_suite.tests.items) |*test_case| { test_case.status = .skipped; try self.results.addTest(test_case); + try events.emit(.{ .test_finished = test_case }); } for (test_suite.suites.items) |nested_suite| { - try self.skipAllTests(nested_suite); + try self.skipAllTests(nested_suite, events); } } + + fn policy(self: *const Self) pipeline.ExecutionPolicy { + return .{ + .bail = self.options.bail, + .filter = self.options.filter, + .timeout_ms = self.options.timeout_ms, + }; + } }; /// Helper function to run tests with default options @@ -290,6 +301,7 @@ test "RunnerOptions default values" { try std.testing.expectEqual(@as(?[]const u8, null), options.filter); try std.testing.expectEqual(ReporterType.spec, options.reporter_type); try std.testing.expectEqual(true, options.use_colors); + try std.testing.expectEqual(@as(?u64, null), options.timeout_ms); } test "RunnerOptions custom values" { diff --git a/tests/reporter_test.zig b/tests/reporter_test.zig index 3f4547e..89c4657 100644 --- a/tests/reporter_test.zig +++ b/tests/reporter_test.zig @@ -69,3 +69,19 @@ test "reporter json output" { try std.testing.expect(std.mem.indexOf(u8, output, "\"name\":\"passes\"") != null); try std.testing.expect(std.mem.indexOf(u8, output, "\"passed\":1") != null); } + +test "reporter set selects the same built-ins for every executor" { + var buffer: [1024]u8 = undefined; + const writer: std.Io.Writer = .fixed(&buffer); + var reporters = ztf.ReporterSet.init( + std.testing.allocator, + writer, + .tap, + "test-results.xml", + false, + ); + defer reporters.deinit(); + + try std.testing.expect(reporters.selected() == &reporters.tap.reporter); + try std.testing.expect(!reporters.selected().use_colors); +}