From 3f37aca2dd8944674060ff702d0b695591804c93 Mon Sep 17 00:00:00 2001 From: glennmichael123 Date: Wed, 30 Sep 2026 01:58:09 +0800 Subject: [PATCH 1/2] feat: add retries repeats and flaky outcomes --- src/cli.zig | 70 ++++++++++++ src/config.zig | 11 ++ src/lib.zig | 1 + src/main.zig | 6 + src/reporter.zig | 132 +++++++++++++++++++++- src/suite.zig | 31 ++++++ src/test_loader.zig | 80 +++++++++++--- src/test_runner.zig | 239 ++++++++++++++++++++++++++++++++++++---- tests/reporter_test.zig | 42 +++++++ 9 files changed, 567 insertions(+), 45 deletions(-) diff --git a/src/cli.zig b/src/cli.zig index 8735d14..8f3a59f 100644 --- a/src/cli.zig +++ b/src/cli.zig @@ -45,6 +45,10 @@ pub const CLIOptions = struct { junit_output: ?[]const u8 = null, // Timeout options timeout: ?u64 = null, // Global timeout in milliseconds + // Retry and repetition options + retries: usize = 0, + repeat: usize = 1, + fail_on_flaky: bool = false, }; pub const version = build_options.version; @@ -75,6 +79,9 @@ pub const CLI = struct { self.options.no_recursive = !config.test_options.recursive; self.options.filter = config.test_options.filter; self.options.timeout = config.test_options.timeout; + self.options.retries = config.test_options.retries; + self.options.repeat = config.test_options.repeat; + self.options.fail_on_flaky = config.test_options.fail_on_flaky; self.options.shard_index = config.sharding.index; self.options.shard_count = config.sharding.count; self.options.parallel = config.parallel.enabled; @@ -107,6 +114,32 @@ pub const CLI = struct { self.options.version = true; } else if (std.mem.eql(u8, arg, "--bail") or std.mem.eql(u8, arg, "-b")) { self.options.bail = true; + } else if (std.mem.eql(u8, arg, "--retry") or std.mem.eql(u8, arg, "--retries")) { + if (i + 1 >= args.len) { + std.debug.print("Error: {s} requires a value\n", .{arg}); + return CLIError.MissingValue; + } + i += 1; + self.options.retries = std.fmt.parseInt(usize, args[i], 10) catch { + std.debug.print("Error: {s} must be a valid number\n", .{arg}); + return CLIError.InvalidArgument; + }; + } else if (std.mem.eql(u8, arg, "--repeat")) { + if (i + 1 >= args.len) { + std.debug.print("Error: --repeat requires a value\n", .{}); + return CLIError.MissingValue; + } + i += 1; + self.options.repeat = std.fmt.parseInt(usize, args[i], 10) catch { + std.debug.print("Error: --repeat must be a valid positive number\n", .{}); + return CLIError.InvalidArgument; + }; + if (self.options.repeat == 0) { + std.debug.print("Error: --repeat must be greater than zero\n", .{}); + return CLIError.InvalidArgument; + } + } else if (std.mem.eql(u8, arg, "--fail-on-flaky")) { + self.options.fail_on_flaky = true; } else if (std.mem.eql(u8, arg, "--verbose")) { self.options.verbose = true; } else if (std.mem.eql(u8, arg, "--quiet") or std.mem.eql(u8, arg, "-q")) { @@ -351,6 +384,9 @@ pub const CLI = struct { \\ -h, --help Show this help message \\ -v, --version Show version information \\ -b, --bail Stop test execution on first failure + \\ --retry Retry each failed test up to N times + \\ --repeat Require N successful runs of each selected test + \\ --fail-on-flaky Exit unsuccessfully when a retry recovers a failure \\ --filter Run only tests matching pattern \\ --grep Same as --filter (alias) \\ --reporter Set reporter type (spec, dot, json, tap, junit) @@ -414,6 +450,8 @@ pub const CLI = struct { \\ zig-test --update-snapshots Update all snapshots \\ zig-test --watch Run in watch mode \\ zig-test --timeout 10000 Set 10s timeout for all tests + \\ zig-test --retry 2 --fail-on-flaky Detect and reject flaky tests + \\ zig-test --repeat 100 --filter parser Stress-run selected tests \\ zig-test --profile-memory --fail-on-leak \\ zig-test --parallel --jobs 4 Run with 4 parallel workers \\ zig-test --config zig-test.json Load config from file @@ -441,6 +479,9 @@ pub const CLI = struct { .n_jobs = self.options.jobs, .junit_output = self.options.junit_output orelse "test-results.xml", .timeout_ms = self.options.timeout, + .retries = self.options.retries, + .repeat = self.options.repeat, + .fail_on_flaky = self.options.fail_on_flaky, }; } }; @@ -662,6 +703,35 @@ test "CLI parse timeout" { try std.testing.expectEqual(@as(?u64, 5000), cli.options.timeout); } +test "CLI parses retry repeat and flaky exit policy" { + var cli = CLI.init(std.testing.allocator); + const args = [_][]const u8{ + "zig-test", + "--retry", + "2", + "--repeat", + "5", + "--fail-on-flaky", + }; + try cli.parse(&args); + try std.testing.expectEqual(@as(usize, 2), cli.options.retries); + try std.testing.expectEqual(@as(usize, 5), cli.options.repeat); + try std.testing.expect(cli.options.fail_on_flaky); + + const runner_options = cli.toRunnerOptions(); + try std.testing.expectEqual(@as(usize, 2), runner_options.retries); + try std.testing.expectEqual(@as(usize, 5), runner_options.repeat); + try std.testing.expect(runner_options.fail_on_flaky); +} + +test "CLI rejects a zero repeat count" { + var cli = CLI.init(std.testing.allocator); + try std.testing.expectError( + CLIError.InvalidArgument, + cli.parse(&.{ "zig-test", "--repeat", "0" }), + ); +} + test "CLI rejects invalid coverage tool" { var cli = CLI.init(std.testing.allocator); const args = [_][]const u8{ "zig-test", "--coverage-tool", "unknown" }; diff --git a/src/config.zig b/src/config.zig index 777c620..836c970 100644 --- a/src/config.zig +++ b/src/config.zig @@ -29,6 +29,9 @@ pub const TestOptions = struct { recursive: bool = true, filter: ?[]const u8 = null, timeout: ?u64 = null, + retries: usize = 0, + repeat: usize = 1, + fail_on_flaky: bool = false, }; pub const ParallelOptions = struct { @@ -86,6 +89,7 @@ pub const ConfigError = error{ InvalidJobs, InvalidPort, InvalidTimeout, + InvalidRepeat, InvalidDebounce, InvalidShard, }; @@ -183,6 +187,7 @@ fn validate(config: TestConfig) !void { if (config.test_options.timeout) |timeout| { if (timeout == 0) return ConfigError.InvalidTimeout; } + if (config.test_options.repeat == 0) return ConfigError.InvalidRepeat; if (config.watch.debounce_ms == 0) return ConfigError.InvalidDebounce; const has_shard_index = config.sharding.index != null; const has_shard_count = config.sharding.count != null; @@ -199,6 +204,9 @@ test "TestConfig defaults match CLI discovery defaults" { try std.testing.expectEqualStrings(".", config.test_options.test_dir); try std.testing.expect(config.test_options.recursive); try std.testing.expect(config.test_options.timeout == null); + try std.testing.expectEqual(@as(usize, 0), config.test_options.retries); + try std.testing.expectEqual(@as(usize, 1), config.test_options.repeat); + try std.testing.expect(!config.test_options.fail_on_flaky); try std.testing.expect(config.sharding.index == null); try std.testing.expect(config.sharding.count == null); try std.testing.expect(!config.parallel.enabled); @@ -219,6 +227,9 @@ test "configuration validation rejects invalid runtime values" { invalid.parallel.jobs = 0; try std.testing.expectError(ConfigError.InvalidJobs, validate(invalid)); invalid.parallel.jobs = null; + invalid.test_options.repeat = 0; + try std.testing.expectError(ConfigError.InvalidRepeat, validate(invalid)); + invalid.test_options.repeat = 1; invalid.sharding.index = 2; try std.testing.expectError(ConfigError.InvalidShard, validate(invalid)); invalid.sharding.count = 1; diff --git a/src/lib.zig b/src/lib.zig index 195ff36..d21ad13 100644 --- a/src/lib.zig +++ b/src/lib.zig @@ -60,6 +60,7 @@ pub const describeTimeout = suite.describeTimeout; pub const TestSuite = suite.TestSuite; pub const TestCase = suite.TestCase; pub const TestStatus = suite.TestStatus; +pub const TestAttempt = suite.TestAttempt; pub const TestRegistry = suite.TestRegistry; pub const TestFn = suite.TestFn; pub const HookFn = suite.HookFn; diff --git a/src/main.zig b/src/main.zig index 73075b6..8a98efe 100644 --- a/src/main.zig +++ b/src/main.zig @@ -221,6 +221,9 @@ pub fn main(init: std.process.Init.Minimal) !void { .reporter_type = cli_parser.options.reporter, .junit_output = cli_parser.options.junit_output orelse "test-results.xml", .timeout_ms = cli_parser.options.timeout, + .retries = cli_parser.options.retries, + .repeat = cli_parser.options.repeat, + .fail_on_flaky = cli_parser.options.fail_on_flaky, .reporter_writer = &stdout_writer.interface, .coverage_options = cov_opts, .ui_server = if (ui_server) |*server| server else null, @@ -264,6 +267,9 @@ pub fn main(init: std.process.Init.Minimal) !void { .reporter_type = cli_parser.options.reporter, .junit_output = cli_parser.options.junit_output orelse "test-results.xml", .timeout_ms = cli_parser.options.timeout, + .retries = cli_parser.options.retries, + .repeat = cli_parser.options.repeat, + .fail_on_flaky = cli_parser.options.fail_on_flaky, .reporter_writer = &stdout_writer.interface, .coverage_options = cov_opts, .ui_server = if (ui_server) |*server| server else null, diff --git a/src/reporter.zig b/src/reporter.zig index 0d2148a..bbe375c 100644 --- a/src/reporter.zig +++ b/src/reporter.zig @@ -71,21 +71,25 @@ pub const Reporter = struct { pub const TestResults = struct { total: usize = 0, passed: usize = 0, + flaky: usize = 0, failed: usize = 0, skipped: usize = 0, total_time_ns: u64 = 0, failed_tests: std.ArrayList(suite.TestCase), + flaky_tests: std.ArrayList(suite.TestCase), allocator: std.mem.Allocator, pub fn init(allocator: std.mem.Allocator) TestResults { return TestResults{ .failed_tests = .empty, + .flaky_tests = .empty, .allocator = allocator, }; } pub fn deinit(self: *TestResults) void { self.failed_tests.deinit(self.allocator); + self.flaky_tests.deinit(self.allocator); } pub fn addTest(self: *TestResults, test_case: *const suite.TestCase) !void { @@ -94,6 +98,10 @@ pub const TestResults = struct { switch (test_case.status) { .passed => self.passed += 1, + .flaky => { + self.flaky += 1; + try self.flaky_tests.append(self.allocator, test_case.*); + }, .failed => { self.failed += 1; try self.failed_tests.append(self.allocator, test_case.*); @@ -188,6 +196,9 @@ pub const SpecReporter = struct { try s.output().print("{s}Test Summary:{s}\n", .{ Colors.bold, Colors.reset }); try s.output().print(" Total: {d}\n", .{results.total}); try s.output().print(" {s}Passed: {d}{s}\n", .{ Colors.green, results.passed, Colors.reset }); + if (results.flaky > 0) { + try s.output().print(" {s}Flaky: {d}{s}\n", .{ Colors.yellow, results.flaky, Colors.reset }); + } if (results.failed > 0) { try s.output().print(" {s}Failed: {d}{s}\n", .{ Colors.red, results.failed, Colors.reset }); } @@ -199,6 +210,9 @@ pub const SpecReporter = struct { try s.output().print("Test Summary:\n", .{}); try s.output().print(" Total: {d}\n", .{results.total}); try s.output().print(" Passed: {d}\n", .{results.passed}); + if (results.flaky > 0) { + try s.output().print(" Flaky: {d}\n", .{results.flaky}); + } if (results.failed > 0) { try s.output().print(" Failed: {d}\n", .{results.failed}); } @@ -265,6 +279,25 @@ pub const SpecReporter = struct { try s.output().print("✓ {s} ({d:.2}ms)\n", .{ test_case.name, time_ms }); } }, + .flaky => { + if (reporter.use_colors) { + try s.output().print("{s}~{s} {s} {s}({d} attempts, {d:.2}ms){s}\n", .{ + Colors.yellow, + Colors.reset, + test_case.name, + Colors.gray, + test_case.attempts.items.len, + time_ms, + Colors.reset, + }); + } else { + try s.output().print("~ {s} (flaky after {d} attempts, {d:.2}ms)\n", .{ + test_case.name, + test_case.attempts.items.len, + time_ms, + }); + } + }, .failed => { if (reporter.use_colors) { try s.output().print("{s}✗{s} {s} {s}({d:.2}ms){s}\n", .{ @@ -353,11 +386,13 @@ pub const DotReporter = struct { if (reporter.use_colors) { try s.output().print("{s}Passed: {d}{s}, ", .{ Colors.green, results.passed, Colors.reset }); + try s.output().print("{s}Flaky: {d}{s}, ", .{ Colors.yellow, results.flaky, Colors.reset }); try s.output().print("{s}Failed: {d}{s}, ", .{ Colors.red, results.failed, Colors.reset }); try s.output().print("Total: {d} ({d:.2}ms)\n", .{ results.total, total_time_ms }); } else { - try s.output().print("Passed: {d}, Failed: {d}, Total: {d} ({d:.2}ms)\n", .{ + try s.output().print("Passed: {d}, Flaky: {d}, Failed: {d}, Total: {d} ({d:.2}ms)\n", .{ results.passed, + results.flaky, results.failed, results.total, total_time_ms, @@ -391,6 +426,13 @@ pub const DotReporter = struct { try s.output().print(".", .{}); } }, + .flaky => { + if (reporter.use_colors) { + try s.output().print("{s}~{s}", .{ Colors.yellow, Colors.reset }); + } else { + try s.output().print("~", .{}); + } + }, .failed => { if (reporter.use_colors) { try s.output().print("{s}F{s}", .{ Colors.red, Colors.reset }); @@ -422,6 +464,7 @@ pub const JsonReporter = struct { writer: std.Io.Writer, writer_ref: ?*std.Io.Writer = null, suites: std.ArrayList([]const u8), + emitted_tests: bool = false, const Self = @This(); @@ -472,9 +515,10 @@ pub const JsonReporter = struct { const s = self(reporter); const total_time_ms = @as(f64, @floatFromInt(results.total_time_ns)) / 1_000_000.0; - try s.output().print("],\"summary\":{{\"total\":{d},\"passed\":{d},\"failed\":{d},\"skipped\":{d},\"time\":{d:.2}}}}}\n", .{ + try s.output().print("\n],\"summary\":{{\"total\":{d},\"passed\":{d},\"flaky\":{d},\"failed\":{d},\"skipped\":{d},\"time\":{d:.2}}}}}\n", .{ results.total, results.passed, + results.flaky, results.failed, results.skipped, total_time_ms, @@ -505,11 +549,15 @@ pub const JsonReporter = struct { const status_str = switch (test_case.status) { .passed => "passed", + .flaky => "flaky", .failed => "failed", .skipped => "skipped", else => "unknown", }; + if (s.emitted_tests) try s.output().print(",\n", .{}); + s.emitted_tests = true; + // Note: In a real implementation, you'd want to properly escape JSON strings try s.output().print(" {{\"name\":\"{s}\",\"status\":\"{s}\",\"time\":{d:.2}", .{ test_case.name, @@ -521,7 +569,26 @@ pub const JsonReporter = struct { try s.output().print(",\"error\":\"{s}\"", .{msg}); } - try s.output().print("}},\n", .{}); + try s.output().print(",\"attempts\":[", .{}); + for (test_case.attempts.items, 0..) |attempt, index| { + if (index > 0) try s.output().print(",", .{}); + const attempt_status = switch (attempt.status) { + .passed => "passed", + .flaky => "flaky", + .failed => "failed", + .skipped => "skipped", + else => "unknown", + }; + try s.output().print( + "{{\"number\":{d},\"repetition\":{d},\"status\":\"{s}\",\"durationNs\":{d}", + .{ attempt.number, attempt.repetition, attempt_status, attempt.duration_ns }, + ); + if (attempt.error_message) |message| { + try s.output().print(",\"error\":\"{s}\"", .{message}); + } + try s.output().print("}}", .{}); + } + try s.output().print("]}}", .{}); } }; @@ -593,6 +660,7 @@ pub const TAPReporter = struct { const status = switch (test_case.status) { .passed => "ok", + .flaky => "ok", .failed => "not ok", .skipped => "ok", else => "not ok", @@ -600,7 +668,9 @@ pub const TAPReporter = struct { try self.output().print("{s} {d} - {s}", .{ status, self.test_count, test_case.name }); - if (test_case.status == .skipped) { + if (test_case.status == .flaky) { + try self.output().print(" # FLAKY attempts={d}\n", .{test_case.attempts.items.len}); + } else if (test_case.status == .skipped) { try self.output().print(" # SKIP\n", .{}); } else if (test_case.status == .failed and test_case.error_message != null) { try self.output().print("\n ---\n message: {s}\n ...\n", .{test_case.error_message.?}); @@ -631,6 +701,10 @@ pub const JUnitReporter = struct { if (test_case.error_message) |msg| { allocator.free(msg); } + for (test_case.attempts) |attempt| { + if (attempt.error_message) |msg| allocator.free(msg); + } + allocator.free(test_case.attempts); } self.tests.deinit(allocator); allocator.free(self.name); @@ -642,6 +716,15 @@ pub const JUnitReporter = struct { time: f64, status: suite.TestStatus, error_message: ?[]const u8, + attempts: []AttemptResult, + }; + + const AttemptResult = struct { + number: usize, + repetition: usize, + status: suite.TestStatus, + duration_ns: u64, + error_message: ?[]const u8, }; pub fn init(allocator: std.mem.Allocator, output_file: []const u8) Self { @@ -716,11 +799,27 @@ pub const JUnitReporter = struct { else null; + const attempts = try self.allocator.alloc(AttemptResult, test_case.attempts.items.len); + errdefer self.allocator.free(attempts); + for (test_case.attempts.items, 0..) |attempt, index| { + attempts[index] = .{ + .number = attempt.number, + .repetition = attempt.repetition, + .status = attempt.status, + .duration_ns = attempt.duration_ns, + .error_message = if (attempt.error_message) |message| + try self.allocator.dupe(u8, message) + else + null, + }; + } + const test_result = TestCaseResult{ .name = try self.allocator.dupe(u8, test_case.name), .time = time_seconds, .status = test_case.status, .error_message = error_msg, + .attempts = attempts, }; try test_suite.tests.append(self.allocator, test_result); @@ -732,10 +831,11 @@ pub const JUnitReporter = struct { defer buffer.deinit(self.allocator); try buffer.appendSlice(self.allocator, "\n"); - try buffer.print(self.allocator, "\n", .{ + try buffer.print(self.allocator, "\n", .{ results.total, results.failed, results.skipped, + results.flaky, }); for (self.suites.items) |suite_result| { @@ -768,11 +868,16 @@ pub const JUnitReporter = struct { try buffer.print(self.allocator, " \n", .{ test_result.error_message orelse "Test failed", }); + try writeAttemptHistory(&buffer, self.allocator, test_result.attempts); try buffer.appendSlice(self.allocator, " \n"); } else if (test_result.status == .skipped) { try buffer.appendSlice(self.allocator, ">\n"); try buffer.appendSlice(self.allocator, " \n"); try buffer.appendSlice(self.allocator, " \n"); + } else if (test_result.attempts.len > 1) { + try buffer.appendSlice(self.allocator, ">\n"); + try writeAttemptHistory(&buffer, self.allocator, test_result.attempts); + try buffer.appendSlice(self.allocator, " \n"); } else { try buffer.appendSlice(self.allocator, "/>\n"); } @@ -785,6 +890,23 @@ pub const JUnitReporter = struct { try compat.writeFile(self.allocator, self.output_file, buffer.items); } + + fn writeAttemptHistory( + buffer: *std.ArrayList(u8), + allocator: std.mem.Allocator, + attempts: []const AttemptResult, + ) !void { + try buffer.appendSlice(allocator, " "); + for (attempts, 0..) |attempt, index| { + if (index > 0) try buffer.appendSlice(allocator, " "); + try buffer.print( + allocator, + "attempt={d} repetition={d} status={s} duration_ns={d}", + .{ attempt.number, attempt.repetition, @tagName(attempt.status), attempt.duration_ns }, + ); + } + try buffer.appendSlice(allocator, "\n"); + } }; /// Owns every built-in reporter and selects one through a shared interface. diff --git a/src/suite.zig b/src/suite.zig index d81b1ca..ffe4c67 100644 --- a/src/suite.zig +++ b/src/suite.zig @@ -24,10 +24,21 @@ pub const TestStatus = enum { pending, running, passed, + flaky, failed, skipped, }; +/// The outcome of one execution of a test. A logical test may have several +/// attempts when retries or repeated runs are enabled. +pub const TestAttempt = struct { + number: usize, + repetition: usize, + status: TestStatus, + duration_ns: u64, + error_message: ?[]const u8 = null, +}; + /// Individual test case pub const TestCase = struct { name: []const u8, @@ -37,6 +48,9 @@ pub const TestCase = struct { status: TestStatus = .pending, error_message: ?[]const u8 = null, execution_time_ns: u64 = 0, + /// Override the runner retry count for this test when non-null. + retry_count: ?usize = null, + attempts: std.ArrayList(TestAttempt) = .empty, skip: bool = false, only: bool = false, file: []const u8 = "", @@ -65,6 +79,20 @@ pub const TestCase = struct { .timeout_ms = timeout_ms, }; } + + /// Configure an additional-attempt count for this individual test. + pub fn withRetries(self: TestCase, retries: usize) TestCase { + var configured = self; + configured.retry_count = retries; + return configured; + } + + pub fn deinit(self: *TestCase, allocator: std.mem.Allocator) void { + for (self.attempts.items) |attempt| { + if (attempt.error_message) |message| allocator.free(message); + } + self.attempts.deinit(allocator); + } }; /// Test suite (describe block) @@ -103,6 +131,7 @@ pub const TestSuite = struct { for (self.suites.items) |suite| { suite.deinit(); } + for (self.tests.items) |*test_case| test_case.deinit(self.allocator); self.tests.deinit(self.allocator); self.suites.deinit(self.allocator); self.before_each_hooks.deinit(self.allocator); @@ -456,12 +485,14 @@ test "TestStatus enum values" { const pending_status = TestStatus.pending; const running_status = TestStatus.running; const passed_status = TestStatus.passed; + const flaky_status = TestStatus.flaky; const failed_status = TestStatus.failed; const skipped_status = TestStatus.skipped; try std.testing.expect(pending_status == .pending); try std.testing.expect(running_status == .running); try std.testing.expect(passed_status == .passed); + try std.testing.expect(flaky_status == .flaky); try std.testing.expect(failed_status == .failed); try std.testing.expect(skipped_status == .skipped); } diff --git a/src/test_loader.zig b/src/test_loader.zig index be5943f..67f2727 100644 --- a/src/test_loader.zig +++ b/src/test_loader.zig @@ -26,6 +26,12 @@ pub const LoaderOptions = struct { junit_output: []const u8 = "test-results.xml", /// Global per-file timeout checked by the shared execution policy timeout_ms: ?u64 = null, + /// Additional process attempts after a failed test file. + retries: usize = 0, + /// Number of successful repetitions required for every selected file. + repeat: usize = 1, + /// Treat recovered failures as an unsuccessful run. + fail_on_flaky: bool = false, /// Optional output supplied by a CLI host. Embedded/test callers default /// to stderr so they do not interfere with Zig's stdout test protocol. reporter_writer: ?*std.Io.Writer = null, @@ -85,6 +91,11 @@ pub fn runDiscoveredTests( }; var results = reporter.TestResults.init(allocator); defer results.deinit(); + var executed_cases: std.ArrayList(suite.TestCase) = .empty; + defer { + for (executed_cases.items) |*test_case| test_case.deinit(allocator); + executed_cases.deinit(allocator); + } try events.emit(.{ .run_started = selected_files }); @@ -120,23 +131,49 @@ pub fn runDiscoveredTests( try server.broadcast("test_start", json); } - const start_time = compat.nanoTimestamp(); - var result = try runTestFile(allocator, file.path, options); - const end_time = compat.nanoTimestamp(); - const execution_time_ns: u64 = @intCast(end_time - start_time); - const timed_out = policy.timedOut(execution_time_ns); - if (timed_out) result = false; - files_run += 1; var test_case = suite.TestCase.init(file.name, externalTestNoop); test_case.file = file.relative_path; - test_case.execution_time_ns = execution_time_ns; - test_case.status = if (result) .passed else .failed; - if (timed_out) { - test_case.error_message = "External test file exceeded the configured timeout"; - } else if (!result) { - test_case.error_message = "External zig test process failed"; + const repeat_count = @max(options.repeat, 1); + var attempt_number: usize = 0; + var had_failure = false; + var final_failure = false; + + repetitions: for (1..repeat_count + 1) |repetition| { + for (1..options.retries + 2) |attempt_in_repetition| { + attempt_number += 1; + const start_time = compat.nanoTimestamp(); + var passed = try runTestFile(allocator, file.path, options); + const duration_ns: u64 = @intCast(compat.nanoTimestamp() - start_time); + const timed_out = policy.timedOut(duration_ns); + if (timed_out) passed = false; + const error_message = if (timed_out) + try allocator.dupe(u8, "External test file exceeded the configured timeout") + else if (!passed) + try allocator.dupe(u8, "External zig test process failed") + else + null; + try test_case.attempts.append(allocator, .{ + .number = attempt_number, + .repetition = repetition, + .status = if (passed) .passed else .failed, + .duration_ns = duration_ns, + .error_message = error_message, + }); + test_case.execution_time_ns += duration_ns; + + if (passed) break; + had_failure = true; + test_case.error_message = error_message; + if (attempt_in_repetition == options.retries + 1) { + final_failure = true; + break :repetitions; + } + } } + test_case.status = if (final_failure) .failed else if (had_failure) .flaky else .passed; + if (!final_failure) test_case.error_message = null; + try executed_cases.append(allocator, test_case); try results.addTest(&test_case); try events.emit(.{ .test_finished = &test_case }); try events.emit(.{ .suite_finished = file.relative_path }); @@ -144,14 +181,20 @@ pub fn runDiscoveredTests( // Notify UI of test file end if (options.ui_server) |server| { var buffer: [512]u8 = undefined; - const status = if (result) "passed" else "failed"; - const json = try std.fmt.bufPrint(&buffer, "{{\"name\":\"{s}\",\"status\":\"{s}\",\"execution_time_ns\":{d},\"error_message\":\"\"}}", .{ file.name, status, execution_time_ns }); + const json = try std.fmt.bufPrint(&buffer, "{{\"name\":\"{s}\",\"status\":\"{s}\",\"execution_time_ns\":{d},\"attempts\":{d},\"error_message\":\"\"}}", .{ + file.name, + @tagName(test_case.status), + test_case.execution_time_ns, + test_case.attempts.items.len, + }); try server.broadcast("test_end", json); try server.broadcast("suite_end", try std.fmt.bufPrint(&buffer, "{{\"name\":\"{s}\"}}", .{file.name})); } - if (result) { + if (test_case.status == .passed) { if (options.verbose) std.debug.print(" ✓ {s} passed\n", .{file.name}); + } else if (test_case.status == .flaky) { + std.debug.print(" ~ {s} flaky after {d} attempts\n", .{ file.name, test_case.attempts.items.len }); } else { std.debug.print(" ✗ {s} failed\n", .{file.name}); if (policy.shouldStop(&results)) { @@ -188,7 +231,7 @@ pub fn runDiscoveredTests( try events.emit(.{ .run_finished = &results }); try reporters.flush(); - return results.failed == 0; + return results.failed == 0 and (!options.fail_on_flaky or results.flaky == 0); } fn externalTestNoop(_: std.mem.Allocator) !void {} @@ -292,6 +335,9 @@ test "LoaderOptions default values" { try std.testing.expectEqual(reporter.ReporterType.spec, options.reporter_type); try std.testing.expectEqualStrings("test-results.xml", options.junit_output); try std.testing.expectEqual(@as(?u64, null), options.timeout_ms); + try std.testing.expectEqual(@as(usize, 0), options.retries); + try std.testing.expectEqual(@as(usize, 1), options.repeat); + try std.testing.expect(!options.fail_on_flaky); try std.testing.expectEqual(@as(?*std.Io.Writer, null), options.reporter_writer); try std.testing.expectEqual(@as(?coverage.CoverageOptions, null), options.coverage_options); } diff --git a/src/test_runner.zig b/src/test_runner.zig index 5c57a86..82b0bd5 100644 --- a/src/test_runner.zig +++ b/src/test_runner.zig @@ -19,6 +19,12 @@ pub const RunnerOptions = struct { n_jobs: ?usize = null, // Number of parallel jobs junit_output: []const u8 = "test-results.xml", timeout_ms: ?u64 = null, + /// Additional attempts after a failed execution. + retries: usize = 0, + /// Number of times every selected test is required to pass. + repeat: usize = 1, + /// Treat tests that pass after a retry as an unsuccessful run. + fail_on_flaky: bool = false, /// Optional output supplied by a CLI host. Embedded/test callers default /// to stderr so they do not interfere with Zig's stdout test protocol. reporter_writer: ?*std.Io.Writer = null, @@ -75,7 +81,7 @@ pub const TestRunner = struct { } // Run tests in parallel or sequential based on options - if (self.options.parallel) { + if (self.options.parallel and self.options.retries == 0 and self.options.repeat == 1) { const parallel_opts = parallel.ParallelOptions{ .enabled = true, .n_jobs = self.options.n_jobs, @@ -95,7 +101,8 @@ pub const TestRunner = struct { } try reporters.flush(); - return all_passed and self.results.failed == 0; + return all_passed and self.results.failed == 0 and + (!self.options.fail_on_flaky or self.results.flaky == 0); } // Sequential execution (original behavior) @@ -113,7 +120,8 @@ pub const TestRunner = struct { // Flush output try reporters.flush(); - return self.results.failed == 0; + return self.results.failed == 0 and + (!self.options.fail_on_flaky or self.results.flaky == 0); } fn addSuiteResults(self: *Self, test_suite: *suite.TestSuite) !void { @@ -189,52 +197,88 @@ pub const TestRunner = struct { fn runTest(self: *Self, test_case: *suite.TestCase, test_suite: *suite.TestSuite, events: pipeline.EventStream) !void { try events.emit(.{ .test_started = test_case.name }); + for (test_case.attempts.items) |attempt| { + if (attempt.error_message) |message| self.allocator.free(message); + } + test_case.attempts.clearRetainingCapacity(); + test_case.execution_time_ns = 0; + test_case.error_message = null; + + const retries = test_case.retry_count orelse self.options.retries; + const repeat_count = @max(self.options.repeat, 1); + var attempt_number: usize = 0; + var had_failure = false; + var final_failure = false; + + repetitions: for (1..repeat_count + 1) |repetition| { + for (1..retries + 2) |attempt_in_repetition| { + attempt_number += 1; + const passed = try self.runSingleAttempt(test_case, test_suite, repetition, attempt_number); + if (passed) break; + + had_failure = true; + if (attempt_in_repetition == retries + 1) { + final_failure = true; + break :repetitions; + } + } + } + + if (!final_failure) { + test_case.status = if (had_failure) .flaky else .passed; + test_case.error_message = null; + } + + try self.results.addTest(test_case); + try events.emit(.{ .test_finished = test_case }); + } + + fn runSingleAttempt( + self: *Self, + test_case: *suite.TestCase, + test_suite: *suite.TestSuite, + repetition: usize, + attempt_number: usize, + ) !bool { test_case.status = .running; + test_case.error_message = null; const start_time = compat.nanoTimestamp(); - // Get all beforeEach hooks (including parent hooks) var before_hooks = try test_suite.getAllBeforeEachHooks(self.allocator); defer before_hooks.deinit(self.allocator); - // Run beforeEach hooks var before_failed = false; for (before_hooks.items) |hook| { hook(self.allocator) catch |err| { test_case.status = .failed; - const err_msg = try std.fmt.allocPrint(self.allocator, "beforeEach hook failed: {any}", .{err}); - test_case.error_message = err_msg; + test_case.error_message = try std.fmt.allocPrint( + self.allocator, + "beforeEach hook failed: {any}", + .{err}, + ); before_failed = true; break; }; } - // Run the actual test if beforeEach succeeded if (!before_failed) { test_case.test_fn(self.allocator) catch |err| { test_case.status = .failed; - const err_msg = try std.fmt.allocPrint(self.allocator, "{any}", .{err}); - test_case.error_message = err_msg; + test_case.error_message = try std.fmt.allocPrint(self.allocator, "{any}", .{err}); }; - - if (test_case.status == .running) { - test_case.status = .passed; - } + if (test_case.status == .running) test_case.status = .passed; } - // Get all afterEach hooks (including parent hooks) var after_hooks = try test_suite.getAllAfterEachHooks(self.allocator); defer after_hooks.deinit(self.allocator); - - // Run afterEach hooks (always run, even if test failed) for (after_hooks.items) |hook| { hook(self.allocator) catch |err| { std.debug.print("afterEach hook failed: {any}\n", .{err}); }; } - const end_time = compat.nanoTimestamp(); - test_case.execution_time_ns = @intCast(end_time - start_time); - if (test_case.status == .passed and self.policy().timedOut(test_case.execution_time_ns)) { + const duration_ns: u64 = @intCast(compat.nanoTimestamp() - start_time); + if (test_case.status == .passed and self.policy().timedOut(duration_ns)) { test_case.status = .failed; test_case.error_message = try std.fmt.allocPrint( self.allocator, @@ -242,9 +286,15 @@ pub const TestRunner = struct { .{self.options.timeout_ms.?}, ); } - - try self.results.addTest(test_case); - try events.emit(.{ .test_finished = test_case }); + test_case.execution_time_ns += duration_ns; + try test_case.attempts.append(self.allocator, .{ + .number = attempt_number, + .repetition = repetition, + .status = test_case.status, + .duration_ns = duration_ns, + .error_message = test_case.error_message, + }); + return test_case.status == .passed; } /// Skip all tests in a suite @@ -302,6 +352,9 @@ test "RunnerOptions default values" { try std.testing.expectEqual(ReporterType.spec, options.reporter_type); try std.testing.expectEqual(true, options.use_colors); try std.testing.expectEqual(@as(?u64, null), options.timeout_ms); + try std.testing.expectEqual(@as(usize, 0), options.retries); + try std.testing.expectEqual(@as(usize, 1), options.repeat); + try std.testing.expect(!options.fail_on_flaky); } test "RunnerOptions custom values" { @@ -317,3 +370,143 @@ test "RunnerOptions custom values" { try std.testing.expectEqual(ReporterType.json, options.reporter_type); try std.testing.expectEqual(false, options.use_colors); } + +var flaky_fixture_runs: usize = 0; +var failing_fixture_runs: usize = 0; +var skipped_by_bail_runs: usize = 0; +var repeated_fixture_runs: usize = 0; +var timeout_fixture_runs: usize = 0; + +fn flakyFixture(_: std.mem.Allocator) !void { + flaky_fixture_runs += 1; + if (flaky_fixture_runs == 1) return error.TransientFailure; +} + +fn failingFixture(_: std.mem.Allocator) !void { + failing_fixture_runs += 1; + return error.DeterministicFailure; +} + +fn skippedByBailFixture(_: std.mem.Allocator) !void { + skipped_by_bail_runs += 1; +} + +fn repeatedFixture(_: std.mem.Allocator) !void { + repeated_fixture_runs += 1; +} + +fn timeoutFixture(_: std.mem.Allocator) !void { + timeout_fixture_runs += 1; + compat.sleep(std.time.ns_per_ms); +} + +test "retries record deterministic flaky attempt history" { + flaky_fixture_runs = 0; + const allocator = std.testing.allocator; + var registry = suite.TestRegistry.init(allocator); + defer registry.deinit(); + const test_suite = try suite.TestSuite.init(allocator, "retry suite"); + try test_suite.addTest(suite.TestCase.init("flaky fixture", flakyFixture).withRetries(1)); + try registry.registerSuite(test_suite); + + var output: [4096]u8 = undefined; + var writer: std.Io.Writer = .fixed(&output); + var runner = TestRunner.init(allocator, ®istry, .{ + .reporter_writer = &writer, + .use_colors = false, + }); + defer runner.deinit(); + + try std.testing.expect(try runner.run()); + const test_case = &test_suite.tests.items[0]; + try std.testing.expectEqual(suite.TestStatus.flaky, test_case.status); + try std.testing.expectEqual(@as(usize, 2), test_case.attempts.items.len); + try std.testing.expectEqual(suite.TestStatus.failed, test_case.attempts.items[0].status); + try std.testing.expectEqual(suite.TestStatus.passed, test_case.attempts.items[1].status); + try std.testing.expectEqual(@as(usize, 1), runner.results.flaky); + + var strict_runner = TestRunner.init(allocator, ®istry, .{ + .retries = 1, + .fail_on_flaky = true, + .reporter_writer = &writer, + .use_colors = false, + }); + defer strict_runner.deinit(); + flaky_fixture_runs = 0; + try std.testing.expect(!try strict_runner.run()); +} + +test "repeat mode runs every required passing repetition" { + repeated_fixture_runs = 0; + const allocator = std.testing.allocator; + var registry = suite.TestRegistry.init(allocator); + defer registry.deinit(); + const test_suite = try suite.TestSuite.init(allocator, "repeat suite"); + try test_suite.addTest(suite.TestCase.init("repeated fixture", repeatedFixture)); + try registry.registerSuite(test_suite); + + var output: [4096]u8 = undefined; + var writer: std.Io.Writer = .fixed(&output); + var runner = TestRunner.init(allocator, ®istry, .{ + .repeat = 3, + .reporter_writer = &writer, + .use_colors = false, + }); + defer runner.deinit(); + + try std.testing.expect(try runner.run()); + try std.testing.expectEqual(@as(usize, 3), repeated_fixture_runs); + try std.testing.expectEqual(@as(usize, 3), test_suite.tests.items[0].attempts.items.len); +} + +test "bail waits until retries are exhausted" { + failing_fixture_runs = 0; + skipped_by_bail_runs = 0; + const allocator = std.testing.allocator; + var registry = suite.TestRegistry.init(allocator); + defer registry.deinit(); + const test_suite = try suite.TestSuite.init(allocator, "bail suite"); + try test_suite.addTest(suite.TestCase.init("always fails", failingFixture)); + try test_suite.addTest(suite.TestCase.init("must be skipped", skippedByBailFixture)); + try registry.registerSuite(test_suite); + + var output: [4096]u8 = undefined; + var writer: std.Io.Writer = .fixed(&output); + var runner = TestRunner.init(allocator, ®istry, .{ + .bail = true, + .retries = 2, + .reporter_writer = &writer, + .use_colors = false, + }); + defer runner.deinit(); + + try std.testing.expect(!try runner.run()); + try std.testing.expectEqual(@as(usize, 3), failing_fixture_runs); + try std.testing.expectEqual(@as(usize, 0), skipped_by_bail_runs); + try std.testing.expectEqual(@as(usize, 3), test_suite.tests.items[0].attempts.items.len); +} + +test "timeouts are evaluated for every retry attempt" { + timeout_fixture_runs = 0; + const allocator = std.testing.allocator; + var registry = suite.TestRegistry.init(allocator); + defer registry.deinit(); + const test_suite = try suite.TestSuite.init(allocator, "timeout suite"); + try test_suite.addTest(suite.TestCase.init("times out", timeoutFixture)); + try registry.registerSuite(test_suite); + + var output: [4096]u8 = undefined; + var writer: std.Io.Writer = .fixed(&output); + var runner = TestRunner.init(allocator, ®istry, .{ + .timeout_ms = 0, + .retries = 1, + .reporter_writer = &writer, + .use_colors = false, + }); + defer runner.deinit(); + + try std.testing.expect(!try runner.run()); + try std.testing.expectEqual(@as(usize, 2), timeout_fixture_runs); + try std.testing.expectEqual(@as(usize, 2), test_suite.tests.items[0].attempts.items.len); + try std.testing.expectEqual(suite.TestStatus.failed, test_suite.tests.items[0].status); +} diff --git a/tests/reporter_test.zig b/tests/reporter_test.zig index 89c4657..37b0ffe 100644 --- a/tests/reporter_test.zig +++ b/tests/reporter_test.zig @@ -68,6 +68,48 @@ test "reporter json output" { const output = buffer[0..reporter.writer.end]; try std.testing.expect(std.mem.indexOf(u8, output, "\"name\":\"passes\"") != null); try std.testing.expect(std.mem.indexOf(u8, output, "\"passed\":1") != null); + var parsed = try std.json.parseFromSlice(std.json.Value, std.testing.allocator, output, .{}); + defer parsed.deinit(); +} + +test "reporter json output includes flaky attempt history" { + const allocator = std.testing.allocator; + var buffer: [4096]u8 = undefined; + const writer: std.Io.Writer = .fixed(&buffer); + var reporter = ztf.JsonReporter.init(allocator, writer); + defer reporter.deinit(); + var results = ztf.TestResults.init(allocator); + defer results.deinit(); + results.total = 1; + results.flaky = 1; + var test_case = ztf.TestCase.init("eventually passes", passingTest); + defer test_case.attempts.deinit(allocator); + test_case.status = .flaky; + try test_case.attempts.append(allocator, .{ + .number = 1, + .repetition = 1, + .status = .failed, + .duration_ns = 10, + }); + try test_case.attempts.append(allocator, .{ + .number = 2, + .repetition = 1, + .status = .passed, + .duration_ns = 5, + }); + + try reporter.reporter.onRunStart(1); + try reporter.reporter.onSuiteStart("suite"); + try reporter.reporter.onTestEnd(&test_case); + try reporter.reporter.onSuiteEnd("suite"); + try reporter.reporter.onRunEnd(&results); + + const output = buffer[0..reporter.writer.end]; + try std.testing.expect(std.mem.indexOf(u8, output, "\"status\":\"flaky\"") != null); + try std.testing.expect(std.mem.indexOf(u8, output, "\"attempts\":[") != null); + try std.testing.expect(std.mem.indexOf(u8, output, "\"durationNs\":10") != null); + var parsed = try std.json.parseFromSlice(std.json.Value, allocator, output, .{}); + defer parsed.deinit(); } test "reporter set selects the same built-ins for every executor" { From 552dd738a179c68336122f1618c503139a389978 Mon Sep 17 00:00:00 2001 From: glennmichael123 Date: Wed, 30 Sep 2026 02:00:42 +0800 Subject: [PATCH 2/2] docs: define retry and flaky-test semantics --- CHANGELOG.md | 6 +++ README.md | 22 ++++++++- build.zig | 15 ++++++ docs/execution-pipeline.md | 17 ++++--- docs/retries-and-repeats.md | 92 ++++++++++++++++++++++++++++++++++++ src/config.zig | 3 ++ tests/fixtures/zig-test.json | 5 +- 7 files changed, 152 insertions(+), 8 deletions(-) create mode 100644 docs/retries-and-repeats.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 134da05..43bcc11 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- Retry failed tests and discovered test files with complete attempt history. +- Repeat mode for stress-running selected tests. +- Flaky outcomes, optional fail-on-flaky exit behavior, and JSON/JUnit attempt data. + ### Planned Features - Snapshot testing diff --git a/README.md b/README.md index 771beac..152bb55 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,7 @@ A modern, feature-rich testing framework for Zig inspired by Jest, Vitest, and B - **Code Coverage** - Line, branch, and function coverage with HTML reports (via kcov/grindcov) - **Async Test Support** - Full async test execution with concurrent and sequential modes - **Timeout Handling** - Configurable timeouts at test, suite, and global levels with extension support +- **Retries & Repeats** - Re-run failures, stress-run selected tests, and report flaky outcomes - **Familiar API** - Describe/it syntax similar to Jest and Vitest - **Rich Assertions** - Comprehensive assertion library with `.expect()` and matchers - **Error Assertions** - `toThrow()` and `toThrowError()` for testing error handling @@ -98,6 +99,9 @@ This will automatically discover and run all `*.test.zig` files in the `tests` d - `--reporter ` - Use spec, dot, JSON, TAP, or JUnit output - `--junit-output ` - Set the JUnit XML path - `--timeout ` - Mark files that exceed the global time budget as failed +- `--retry ` - Retry a failed test or discovered test file up to N times +- `--repeat ` - Require N successful repetitions of every selected test +- `--fail-on-flaky` - Return a failing exit status when a retry recovers a failure - `--shard-index ` - Run one one-based file shard - `--shard-count ` - Set the total number of file shards - `--no-color` - Disable color in child Zig test processes @@ -125,6 +129,12 @@ zig-test --test-dir tests --no-recursive # Stop on first failure zig-test --test-dir tests --bail +# Retry failures twice and fail CI if any test is flaky +zig-test --test-dir tests --retry 2 --fail-on-flaky + +# Stress-run matching tests 100 times +zig-test --test-dir tests --filter parser --repeat 100 + # Run the second of four deterministic file shards zig-test --test-dir tests --shard-index 2 --shard-count 4 ``` @@ -168,7 +178,10 @@ precedence over configured values. "test_dir": "tests", "pattern": "*.test.zig", "recursive": true, - "filter": "database" + "filter": "database", + "retries": 2, + "repeat": 10, + "fail_on_flaky": true }, "sharding": { "index": 1, @@ -185,6 +198,9 @@ The accepted top-level sections are `test`, `sharding`, `parallel`, `reporter`, `snapshot`, `watch`, `memory`, `ui`, and `coverage`. Options retain the same execution-mode limitations as their CLI equivalents; unsupported combinations fail explicitly. +See [Retries, repeats, and flaky tests](docs/retries-and-repeats.md) for attempt +semantics, exit codes, reporter output, and programmatic configuration. + ### Parallel execution semantics Programmatic runners can enable bounded execution with `RunnerOptions.parallel` @@ -642,6 +658,9 @@ zig-test --no-color | `--reporter ` | | Set reporter (spec, dot, json, tap, junit) | | `--junit-output ` | | Set the JUnit XML output path | | `--timeout ` | | Set the global test timeout | +| `--retry ` | `--retries ` | Retry each failed test up to N times | +| `--repeat ` | | Require N successful runs of each selected test | +| `--fail-on-flaky` | | Fail the run when a retry recovers a failure | | `--verbose` | | Enable verbose output | | `--quiet` | `-q` | Minimal output | | `--no-color` | | Disable colored output | @@ -921,6 +940,7 @@ MIT License - see [LICENSE](LICENSE) for details. - [x] Memory profiling - [x] Strict JSON configuration files - [x] Parallel test execution +- [x] Retries, repeated runs, and flaky-test reporting - [ ] Parameterized tests (it.each) - [ ] Property-based testing - [ ] IDE integration diff --git a/build.zig b/build.zig index 3a0ab38..fbeed4a 100644 --- a/build.zig +++ b/build.zig @@ -293,6 +293,21 @@ pub fn build(b: *std.Build) void { ); test_step.dependOn(&run_discovery_tap.step); + const run_discovery_repeat = b.addRunArtifact(exe); + run_discovery_repeat.addArgs(&.{ + "--test-dir", + "tests", + "--pattern", + "sample.test.zig", + "--retry", + "1", + "--repeat", + "2", + "--fail-on-flaky", + "--no-color", + }); + test_step.dependOn(&run_discovery_repeat.step); + const run_version = b.addRunArtifact(exe); run_version.addArg("--version"); run_version.expectStdErrEqual(b.fmt("Zig Test Framework v{s}\n", .{package_version})); diff --git a/docs/execution-pipeline.md b/docs/execution-pipeline.md index 239758a..54040ed 100644 --- a/docs/execution-pipeline.md +++ b/docs/execution-pipeline.md @@ -26,10 +26,14 @@ reporters, including the configured JUnit output path. - Registered tests match filters directly by test name. Discovery forwards the same filter to `zig test`, where Zig applies it to names inside each file. - Bail stops scheduling after the shared result model records a failure. Work - already started by a parallel batch cannot be cancelled. + already started by a parallel batch cannot be cancelled. With retries, bail + is evaluated only after the retry budget is exhausted. - A global timeout compares elapsed execution time with the same policy in both - executors. It marks an over-budget result failed after control returns; it - does not yet terminate an in-process function or external child process. + executors on every attempt. It marks an over-budget attempt failed after + control returns; it does not yet terminate an in-process function or external + child process. +- Every logical result owns ordered attempt history. A failure followed by a + successful retry is classified as `flaky`, distinct from an ordinary pass. - Summaries always come from `TestResults`, rather than executor-local counters. ## Extension points @@ -48,10 +52,11 @@ depend on one serialization format. Existing programmatic APIs remain source compatible: - `TestRunner.init`, `runTests`, and `runTestsWithOptions` are unchanged. -- `RunnerOptions` only gains optional `junit_output`, `timeout_ms`, and - `reporter_writer` fields. +- `RunnerOptions` gains optional reporter, timeout, retry, repeat, and flaky-exit + fields with backward-compatible defaults. - `runDiscoveredTests` and `LoaderOptions` remain available; optional reporter, - JUnit path, timeout, and writer fields default to the previous spec behavior. + JUnit path, timeout, retry, repeat, flaky-exit, and writer fields default to + the previous spec behavior. CLI hosts can inject a stdout writer with `reporter_writer`. Embedded callers may leave it unset; reporter output then uses stderr so it cannot corrupt Zig's diff --git a/docs/retries-and-repeats.md b/docs/retries-and-repeats.md new file mode 100644 index 0000000..315c2a9 --- /dev/null +++ b/docs/retries-and-repeats.md @@ -0,0 +1,92 @@ +# Retries, repeats, and flaky tests + +The runner records each execution of a logical test as an attempt. An attempt +contains its one-based number, repetition number, status, duration, and error +message. Registered tests and discovered test files use the same model. + +## Command-line use + +```bash +# One initial attempt plus as many as two retries after failure +zig-test --test-dir tests --retry 2 + +# Require every selected test to pass 100 times +zig-test --test-dir tests --filter parser --repeat 100 + +# Make a recovered failure fail the overall run in CI +zig-test --test-dir tests --retry 2 --fail-on-flaky +``` + +`--retries` is an alias for `--retry`. A retry count is the number of +*additional* attempts after a failure. `--repeat N` requires N successful +repetitions; each repetition receives its own retry budget. Attempt history is +ordered across all repetitions. + +Discovery operates at test-file granularity because each file is an external +`zig test` process. Programmatic mode operates at individual-test granularity. + +## Outcomes and exit status + +- `passed`: every attempt passed without a retry. +- `flaky`: at least one attempt failed, but a retry recovered and all required + repetitions eventually passed. +- `failed`: one repetition exhausted its retry budget. +- `skipped`: the test was not selected for execution. + +Failed tests always make the process exit unsuccessfully. Flaky tests succeed +by default so retries can recover an intermittent failure. `--fail-on-flaky` +changes that policy and returns an unsuccessful exit status when any test is +flaky. + +`--bail` is evaluated after retries, so one failed attempt does not stop the +run. The runner stops scheduling only after a logical test exhausts its retry +budget. The timeout is evaluated independently for every attempt; a timed-out +attempt may be retried. + +When retries or repeats are configured for registered tests, execution is +sequential even if parallel mode was requested. This preserves deterministic +attempt ordering and hook behavior. + +## Programmatic configuration + +Set defaults for a registered-test run with `RunnerOptions`: + +```zig +const options = ztf.RunnerOptions{ + .retries = 2, + .repeat = 10, + .fail_on_flaky = true, +}; +const passed = try ztf.runTestsWithOptions(allocator, registry, options); +``` + +An individual registered test can override the run-level retry count: + +```zig +try suite.addTest( + ztf.TestCase.init("eventually consistent", testEventuallyConsistent) + .withRetries(5), +); +``` + +`LoaderOptions` exposes the same `retries`, `repeat`, and `fail_on_flaky` +fields for callers that run discovered files directly. Strict JSON +configuration uses the corresponding fields in the `test` object: + +```json +{ + "test": { + "retries": 2, + "repeat": 10, + "fail_on_flaky": true + } +} +``` + +## Reporter data + +The spec and dot reporters show flaky results separately from passes. JSON test +objects include an `attempts` array with `number`, `repetition`, `status`, and +`durationNs` fields plus an optional `error`. The JSON summary includes a +`flaky` count. JUnit output includes the flaky count on `` and emits +ordered attempt data in each retried test case's `` element. diff --git a/src/config.zig b/src/config.zig index 836c970..9b044cc 100644 --- a/src/config.zig +++ b/src/config.zig @@ -244,6 +244,9 @@ test "ConfigLoader loads supported JSON and rejects unknown keys" { try std.testing.expectEqualStrings("tests/fixtures", config.test_options.test_dir); try std.testing.expectEqualStrings("selected test passes", config.test_options.filter.?); try std.testing.expect(!config.test_options.recursive); + try std.testing.expectEqual(@as(usize, 1), config.test_options.retries); + try std.testing.expectEqual(@as(usize, 2), config.test_options.repeat); + try std.testing.expect(!config.test_options.fail_on_flaky); try std.testing.expectEqual(@as(?usize, 1), config.sharding.index); try std.testing.expectEqual(@as(?usize, 1), config.sharding.count); diff --git a/tests/fixtures/zig-test.json b/tests/fixtures/zig-test.json index 3ade3a6..93e3c8e 100644 --- a/tests/fixtures/zig-test.json +++ b/tests/fixtures/zig-test.json @@ -3,7 +3,10 @@ "test_dir": "tests/fixtures", "pattern": "filter_fixture.zig", "recursive": false, - "filter": "selected test passes" + "filter": "selected test passes", + "retries": 1, + "repeat": 2, + "fail_on_flaky": false }, "sharding": { "index": 1,