From 514c724ea39257c431b58fe81c17277716288ca4 Mon Sep 17 00:00:00 2001 From: glennmichael123 Date: Wed, 30 Sep 2026 21:39:13 +0800 Subject: [PATCH 1/2] feat: add reproducible randomized test order --- src/cli.zig | 34 ++++++++ src/config.zig | 6 ++ src/event_protocol.zig | 6 +- src/lib.zig | 1 + src/main.zig | 4 + src/parallel.zig | 82 ++++++++++++++++++-- src/random_order.zig | 42 ++++++++++ src/reporter.zig | 31 ++++++-- src/test_history.zig | 19 ++++- src/test_loader.zig | 20 ++++- src/test_runner.zig | 146 +++++++++++++++++++++++++++++++---- src/ui_server.zig | 9 ++- tests/fixtures/zig-test.json | 4 +- tests/reporter_test.zig | 33 ++++++++ 14 files changed, 402 insertions(+), 35 deletions(-) create mode 100644 src/random_order.zig diff --git a/src/cli.zig b/src/cli.zig index 914c027..d6e42a8 100644 --- a/src/cli.zig +++ b/src/cli.zig @@ -49,6 +49,8 @@ pub const CLIOptions = struct { retries: usize = 0, repeat: usize = 1, fail_on_flaky: bool = false, + shuffle: bool = false, + seed: ?u64 = null, }; pub const version = build_options.version; @@ -82,6 +84,8 @@ pub const CLI = struct { self.options.retries = config.test_options.retries; self.options.repeat = config.test_options.repeat; self.options.fail_on_flaky = config.test_options.fail_on_flaky; + self.options.shuffle = config.test_options.shuffle or config.test_options.seed != null; + self.options.seed = config.test_options.seed; self.options.shard_index = config.sharding.index; self.options.shard_count = config.sharding.count; self.options.parallel = config.parallel.enabled; @@ -140,6 +144,19 @@ pub const CLI = struct { } } else if (std.mem.eql(u8, arg, "--fail-on-flaky")) { self.options.fail_on_flaky = true; + } else if (std.mem.eql(u8, arg, "--shuffle")) { + self.options.shuffle = true; + } else if (std.mem.eql(u8, arg, "--seed")) { + if (i + 1 >= args.len) { + std.debug.print("Error: --seed requires a value\n", .{}); + return CLIError.MissingValue; + } + i += 1; + self.options.seed = std.fmt.parseInt(u64, args[i], 10) catch { + std.debug.print("Error: --seed must be a valid unsigned integer\n", .{}); + return CLIError.InvalidArgument; + }; + self.options.shuffle = true; } else if (std.mem.eql(u8, arg, "--verbose")) { self.options.verbose = true; } else if (std.mem.eql(u8, arg, "--quiet") or std.mem.eql(u8, arg, "-q")) { @@ -386,6 +403,8 @@ pub const CLI = struct { \\ --retry Retry each failed test up to N times \\ --repeat Require N successful runs of each selected test \\ --fail-on-flaky Exit unsuccessfully when a retry recovers a failure + \\ --shuffle Randomize suite and test order + \\ --seed Reproduce a randomized order (implies --shuffle) \\ --filter Run only tests matching pattern \\ --grep Same as --filter (alias) \\ --reporter Set reporter type (spec, dot, json, tap, junit) @@ -452,6 +471,8 @@ pub const CLI = struct { \\ zig-test --timeout 10000 Set 10s timeout for all tests \\ zig-test --retry 2 --fail-on-flaky Detect and reject flaky tests \\ zig-test --repeat 100 --filter parser Stress-run selected tests + \\ zig-test --shuffle Run in a generated random order + \\ zig-test --seed 12345 Replay a randomized order \\ zig-test --profile-memory --fail-on-leak \\ zig-test --parallel --jobs 4 Run with 4 parallel workers \\ zig-test --config zig-test.json Load config from file @@ -482,6 +503,8 @@ pub const CLI = struct { .retries = self.options.retries, .repeat = self.options.repeat, .fail_on_flaky = self.options.fail_on_flaky, + .shuffle = self.options.shuffle, + .seed = self.options.seed, }; } }; @@ -724,6 +747,17 @@ test "CLI parses retry repeat and flaky exit policy" { try std.testing.expect(runner_options.fail_on_flaky); } +test "CLI parses shuffle and a seed implies it" { + var cli = CLI.init(std.testing.allocator); + try cli.parse(&.{ "zig-test", "--seed", "12345" }); + try std.testing.expect(cli.options.shuffle); + try std.testing.expectEqual(@as(?u64, 12345), cli.options.seed); + + const runner_options = cli.toRunnerOptions(); + try std.testing.expect(runner_options.shuffle); + try std.testing.expectEqual(@as(?u64, 12345), runner_options.seed); +} + test "CLI rejects a zero repeat count" { var cli = CLI.init(std.testing.allocator); try std.testing.expectError( diff --git a/src/config.zig b/src/config.zig index 9b044cc..dbb4a53 100644 --- a/src/config.zig +++ b/src/config.zig @@ -32,6 +32,8 @@ pub const TestOptions = struct { retries: usize = 0, repeat: usize = 1, fail_on_flaky: bool = false, + shuffle: bool = false, + seed: ?u64 = null, }; pub const ParallelOptions = struct { @@ -207,6 +209,8 @@ test "TestConfig defaults match CLI discovery defaults" { try std.testing.expectEqual(@as(usize, 0), config.test_options.retries); try std.testing.expectEqual(@as(usize, 1), config.test_options.repeat); try std.testing.expect(!config.test_options.fail_on_flaky); + try std.testing.expect(!config.test_options.shuffle); + try std.testing.expect(config.test_options.seed == null); try std.testing.expect(config.sharding.index == null); try std.testing.expect(config.sharding.count == null); try std.testing.expect(!config.parallel.enabled); @@ -247,6 +251,8 @@ test "ConfigLoader loads supported JSON and rejects unknown keys" { try std.testing.expectEqual(@as(usize, 1), config.test_options.retries); try std.testing.expectEqual(@as(usize, 2), config.test_options.repeat); try std.testing.expect(!config.test_options.fail_on_flaky); + try std.testing.expect(config.test_options.shuffle); + try std.testing.expectEqual(@as(?u64, 42), config.test_options.seed); try std.testing.expectEqual(@as(?usize, 1), config.sharding.index); try std.testing.expectEqual(@as(?usize, 1), config.sharding.count); diff --git a/src/event_protocol.zig b/src/event_protocol.zig index 12f2b2f..efd15e8 100644 --- a/src/event_protocol.zig +++ b/src/event_protocol.zig @@ -7,7 +7,10 @@ pub const Status = enum { pending, running, passed, flaky, failed, skipped }; pub const HookKind = enum { before_all, before_each, after_each, after_all }; pub const OutputStream = enum { stdout, stderr }; -pub const RunStart = struct { total: usize }; +pub const RunStart = struct { + total: usize, + random_seed: ?u64 = null, +}; pub const RunEnd = struct { total: usize, passed: usize, @@ -15,6 +18,7 @@ pub const RunEnd = struct { failed: usize, skipped: usize, duration_ns: u64 = 0, + random_seed: ?u64 = null, }; pub const Suite = struct { name: []const u8 }; pub const TestStart = struct { name: []const u8, suite: ?[]const u8 = null }; diff --git a/src/lib.zig b/src/lib.zig index fe23ba1..35426a6 100644 --- a/src/lib.zig +++ b/src/lib.zig @@ -27,6 +27,7 @@ pub const timeout = @import("timeout.zig"); pub const progress = @import("progress.zig"); pub const time = @import("time.zig"); pub const compat = @import("compat.zig"); +pub const random_order = @import("random_order.zig"); // Re-export commonly used types and functions pub const expect = assertions.expect; diff --git a/src/main.zig b/src/main.zig index 7098cdf..746ae41 100644 --- a/src/main.zig +++ b/src/main.zig @@ -189,6 +189,8 @@ pub fn main(init: std.process.Init.Minimal) !void { .retries = cli_parser.options.retries, .repeat = cli_parser.options.repeat, .fail_on_flaky = cli_parser.options.fail_on_flaky, + .shuffle = cli_parser.options.shuffle, + .seed = cli_parser.options.seed, .reporter_writer = &stdout_writer.interface, .coverage_options = cov_opts, .ui_server = if (ui_server) |*server| server else null, @@ -235,6 +237,8 @@ pub fn main(init: std.process.Init.Minimal) !void { .retries = cli_parser.options.retries, .repeat = cli_parser.options.repeat, .fail_on_flaky = cli_parser.options.fail_on_flaky, + .shuffle = cli_parser.options.shuffle, + .seed = cli_parser.options.seed, .reporter_writer = &stdout_writer.interface, .coverage_options = cov_opts, .ui_server = if (ui_server) |*server| server else null, diff --git a/src/parallel.zig b/src/parallel.zig index 380d6a4..da2f0fe 100644 --- a/src/parallel.zig +++ b/src/parallel.zig @@ -3,6 +3,7 @@ const suite = @import("suite.zig"); const reporter = @import("reporter.zig"); const pipeline = @import("pipeline.zig"); const compat = @import("compat.zig"); +const random_order = @import("random_order.zig"); /// Options for bounded parallel test execution. pub const ParallelOptions = struct { @@ -12,6 +13,7 @@ pub const ParallelOptions = struct { enabled: bool = false, filter: ?[]const u8 = null, timeout_ms: ?u64 = null, + random_seed: ?u64 = null, }; pub const ParallelError = error{ @@ -111,6 +113,7 @@ const RunContext = struct { rep: *reporter.Reporter, options: ParallelOptions, worker_limit: usize, + shuffler: ?*random_order.Shuffler, fn runSuite(self: *RunContext, test_suite: *suite.TestSuite) !bool { try self.rep.onSuiteStart(test_suite.name); @@ -131,7 +134,12 @@ const RunContext = struct { var scheduled: std.ArrayList(*suite.TestCase) = .empty; defer scheduled.deinit(self.allocator); - for (test_suite.tests.items) |*test_case| { + var ordered_tests: std.ArrayList(*suite.TestCase) = .empty; + defer ordered_tests.deinit(self.allocator); + for (test_suite.tests.items) |*test_case| try ordered_tests.append(self.allocator, test_case); + if (self.shuffler) |randomizer| randomizer.shuffle(*suite.TestCase, ordered_tests.items); + + for (ordered_tests.items) |test_case| { if (!self.shouldRun(test_suite, test_case)) { test_case.status = .skipped; continue; @@ -143,12 +151,16 @@ const RunContext = struct { try self.runBatch(test_suite, scheduled.items); var all_passed = true; - for (test_suite.tests.items) |*test_case| { + for (ordered_tests.items) |test_case| { try self.rep.onTestEnd(test_case); if (test_case.status == .failed) all_passed = false; } - for (test_suite.suites.items) |nested_suite| { + var nested_suites: std.ArrayList(*suite.TestSuite) = .empty; + defer nested_suites.deinit(self.allocator); + try nested_suites.appendSlice(self.allocator, test_suite.suites.items); + if (self.shuffler) |randomizer| randomizer.shuffle(*suite.TestSuite, nested_suites.items); + for (nested_suites.items) |nested_suite| { if (!try self.runSuite(nested_suite)) all_passed = false; } @@ -201,11 +213,19 @@ const RunContext = struct { } fn reportSkippedSuite(self: *RunContext, test_suite: *suite.TestSuite) !void { - for (test_suite.tests.items) |*test_case| { + var tests: std.ArrayList(*suite.TestCase) = .empty; + defer tests.deinit(self.allocator); + for (test_suite.tests.items) |*test_case| try tests.append(self.allocator, test_case); + if (self.shuffler) |randomizer| randomizer.shuffle(*suite.TestCase, tests.items); + for (tests.items) |test_case| { test_case.status = .skipped; try self.rep.onTestEnd(test_case); } - for (test_suite.suites.items) |nested| { + var nested_suites: std.ArrayList(*suite.TestSuite) = .empty; + defer nested_suites.deinit(self.allocator); + try nested_suites.appendSlice(self.allocator, test_suite.suites.items); + if (self.shuffler) |randomizer| randomizer.shuffle(*suite.TestSuite, nested_suites.items); + for (nested_suites.items) |nested| { try self.rep.onSuiteStart(nested.name); try self.reportSkippedSuite(nested); try self.rep.onSuiteEnd(nested.name); @@ -213,9 +233,9 @@ const RunContext = struct { } }; -/// Run suites in declaration order while executing each suite's direct tests +/// Run suites in the selected deterministic order while executing each suite's direct tests /// with a bounded worker set. Reporter callbacks are serialized and emitted in -/// declaration order after each batch completes. +/// the selected order after each batch completes. pub fn runTestsParallel( allocator: std.mem.Allocator, test_registry: *suite.TestRegistry, @@ -226,23 +246,36 @@ pub fn runTestsParallel( const worker_limit = try resolveWorkerCount(options.n_jobs); const total_tests = test_registry.countAllTests(); + rep.random_seed = options.random_seed; try rep.onRunStart(total_tests); + var shuffler: random_order.Shuffler = undefined; + const shuffler_ptr: ?*random_order.Shuffler = if (options.random_seed) |seed| blk: { + shuffler = .init(seed); + break :blk &shuffler; + } else null; + var context = RunContext{ .allocator = allocator, .registry = test_registry, .rep = rep, .options = options, .worker_limit = worker_limit, + .shuffler = shuffler_ptr, }; + var root_suites: std.ArrayList(*suite.TestSuite) = .empty; + defer root_suites.deinit(allocator); + try root_suites.appendSlice(allocator, test_registry.root_suites.items); + if (shuffler_ptr) |randomizer| randomizer.shuffle(*suite.TestSuite, root_suites.items); var all_passed = true; - for (test_registry.root_suites.items) |test_suite| { + for (root_suites.items) |test_suite| { if (!try context.runSuite(test_suite)) all_passed = false; } var results = reporter.TestResults.init(allocator); defer results.deinit(); + results.random_seed = options.random_seed; for (test_registry.root_suites.items) |test_suite| { try addSuiteResults(&results, test_suite); } @@ -347,6 +380,7 @@ const RecordingReporter = struct { passed: usize = 0, failed: usize = 0, skipped: usize = 0, + random_seed: ?u64 = null, fn init(allocator: std.mem.Allocator) RecordingReporter { return .{ @@ -379,6 +413,7 @@ const RecordingReporter = struct { recording.passed = results.passed; recording.failed = results.failed; recording.skipped = results.skipped; + recording.random_seed = results.random_seed; } fn onSuiteStart(rep: *reporter.Reporter, _: []const u8) !void { @@ -468,6 +503,37 @@ test "parallel reporting is serialized, ordered, and complete" { try std.testing.expectEqual(@as(usize, 1), recording.skipped); } +test "parallel order is reproducible with a fixed seed" { + const allocator = std.heap.smp_allocator; + var registry = suite.TestRegistry.init(allocator); + defer registry.deinit(); + const test_suite = try suite.TestSuite.init(allocator, "randomized"); + inline for (.{ "one", "two", "three", "four", "five", "six" }) |name| { + try test_suite.addTest(suite.TestCase.init(name, passingTest)); + } + try registry.registerSuite(test_suite); + + var first = RecordingReporter.init(allocator); + try std.testing.expect(try runTestsParallel(allocator, ®istry, &first.reporter, .{ + .enabled = true, + .n_jobs = 2, + .random_seed = 8675309, + })); + var second = RecordingReporter.init(allocator); + try std.testing.expect(try runTestsParallel(allocator, ®istry, &second.reporter, .{ + .enabled = true, + .n_jobs = 2, + .random_seed = 8675309, + })); + + try std.testing.expectEqual(first.ended_count, second.ended_count); + for (first.ended_names[0..first.ended_count], second.ended_names[0..second.ended_count]) |a, b| { + try std.testing.expectEqualStrings(a, b); + } + try std.testing.expectEqual(@as(?u64, 8675309), first.random_seed); + try std.testing.expectEqual(@as(?u64, 8675309), second.random_seed); +} + test "parallel executor preserves nested hook lifecycles" { const counters = [_]*std.atomic.Value(usize){ &parent_before_each, diff --git a/src/random_order.zig b/src/random_order.zig new file mode 100644 index 0000000..1ee3915 --- /dev/null +++ b/src/random_order.zig @@ -0,0 +1,42 @@ +const std = @import("std"); +const compat = @import("compat.zig"); + +/// Return the seed that should control this run. Supplying a seed enables +/// shuffling even when `shuffle` is false; otherwise random order is opt-in. +pub fn resolveSeed(shuffle: bool, configured: ?u64) ?u64 { + if (configured) |seed| return seed; + if (!shuffle) return null; + return @truncate(@as(u128, @bitCast(compat.nanoTimestamp()))); +} + +pub const Shuffler = struct { + prng: std.Random.DefaultPrng, + + pub fn init(seed: u64) Shuffler { + return .{ .prng = std.Random.DefaultPrng.init(seed) }; + } + + pub fn shuffle(self: *Shuffler, comptime T: type, items: []T) void { + self.prng.random().shuffleWithIndex(T, items, u64); + } +}; + +test "random order is opt-in and an explicit seed enables it" { + try std.testing.expect(resolveSeed(false, null) == null); + try std.testing.expect(resolveSeed(true, null) != null); + try std.testing.expectEqual(@as(?u64, 0), resolveSeed(false, 0)); + try std.testing.expectEqual(@as(?u64, 42), resolveSeed(true, 42)); +} + +test "the same seed produces the same order" { + var first = [_]u8{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; + var second = first; + var first_shuffler = Shuffler.init(123456789); + var second_shuffler = Shuffler.init(123456789); + + first_shuffler.shuffle(u8, &first); + second_shuffler.shuffle(u8, &second); + + try std.testing.expectEqualSlices(u8, &first, &second); + try std.testing.expect(!std.mem.eql(u8, &first, &[_]u8{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 })); +} diff --git a/src/reporter.zig b/src/reporter.zig index 1b15b96..03b8f7d 100644 --- a/src/reporter.zig +++ b/src/reporter.zig @@ -31,6 +31,7 @@ pub const Reporter = struct { vtable: *const VTable, allocator: std.mem.Allocator, use_colors: bool = true, + random_seed: ?u64 = null, const Self = @This(); @@ -76,6 +77,7 @@ pub const TestResults = struct { failed: usize = 0, skipped: usize = 0, total_time_ns: u64 = 0, + random_seed: ?u64 = null, failed_tests: std.ArrayList(suite.TestCase), flaky_tests: std.ArrayList(suite.TestCase), allocator: std.mem.Allocator, @@ -163,6 +165,9 @@ pub const SpecReporter = struct { } else { try s.output().print("Running {d} test(s)...\n\n", .{total_tests}); } + if (reporter.random_seed) |seed| { + try s.output().print("Random seed: {d}\n\n", .{seed}); + } } fn onRunEnd(reporter: *Reporter, results: *TestResults) !void { @@ -188,6 +193,9 @@ pub const SpecReporter = struct { } } try s.output().print("\n", .{}); + if (results.random_seed) |seed| { + try s.output().print("Reproduce with: --seed {d}\n\n", .{seed}); + } } // Print summary @@ -377,6 +385,7 @@ pub const DotReporter = struct { fn onRunStart(reporter: *Reporter, total_tests: usize) !void { const s = self(reporter); try s.output().print("\nRunning {d} tests:\n", .{total_tests}); + if (reporter.random_seed) |seed| try s.output().print("Random seed: {d}\n", .{seed}); } fn onRunEnd(reporter: *Reporter, results: *TestResults) !void { @@ -399,6 +408,9 @@ pub const DotReporter = struct { total_time_ms, }); } + if (results.failed > 0) { + if (results.random_seed) |seed| try s.output().print("Reproduce with: --seed {d}\n", .{seed}); + } } fn onSuiteStart(reporter: *Reporter, suite_name: []const u8) !void { @@ -512,7 +524,7 @@ pub const JsonReporter = struct { s.emitted_events = false; s.suites.clearRetainingCapacity(); try s.output().writeAll("[\n"); - try s.emit(.{ .run_start = .{ .total = total_tests } }); + try s.emit(.{ .run_start = .{ .total = total_tests, .random_seed = reporter.random_seed } }); } fn onRunEnd(reporter: *Reporter, results: *TestResults) !void { @@ -524,6 +536,7 @@ pub const JsonReporter = struct { .failed = results.failed, .skipped = results.skipped, .duration_ns = results.total_time_ns, + .random_seed = results.random_seed, } }); try s.output().writeAll("\n]\n"); } @@ -618,11 +631,14 @@ pub const TAPReporter = struct { fn onRunStart(reporter: *Reporter, total: usize) !void { const self: *Self = @fieldParentPtr("reporter", reporter); try self.output().print("TAP version 14\n1..{d}\n", .{total}); + if (reporter.random_seed) |seed| try self.output().print("# random-seed {d}\n", .{seed}); } fn onRunEnd(reporter: *Reporter, results: *TestResults) !void { - _ = reporter; - _ = results; + const self: *Self = @fieldParentPtr("reporter", reporter); + if (results.failed > 0) { + if (results.random_seed) |seed| try self.output().print("# reproduce --seed {d}\n", .{seed}); + } } fn onSuiteStart(reporter: *Reporter, suite_name: []const u8) !void { @@ -753,14 +769,13 @@ pub const JUnitReporter = struct { fn onSuiteStart(reporter: *Reporter, suite_name: []const u8) !void { const self: *Self = @fieldParentPtr("reporter", reporter); - const suite_result = try self.allocator.create(TestSuiteResult); - suite_result.* = .{ + const suite_result = TestSuiteResult{ .name = try self.allocator.dupe(u8, suite_name), .tests = .empty, .timestamp = compat.milliTimestamp(), }; - try self.suites.append(self.allocator, suite_result.*); + try self.suites.append(self.allocator, suite_result); self.current_suite = &self.suites.items[self.suites.items.len - 1]; } @@ -843,6 +858,10 @@ pub const JUnitReporter = struct { suite_time, }); + if (results.random_seed) |seed| { + try buffer.print(self.allocator, " \n", .{seed}); + } + for (suite_result.tests.items) |test_result| { try buffer.print(self.allocator, " ") != null); +} From e014cc6d8351ee4c55ae6f6cce6787c03dfd8088 Mon Sep 17 00:00:00 2001 From: glennmichael123 Date: Wed, 30 Sep 2026 21:39:13 +0800 Subject: [PATCH 2/2] docs: explain randomized ordering guarantees --- README.md | 20 +++++++++++-- docs/execution-pipeline.md | 8 +++-- docs/json-event-protocol.md | 6 ++-- docs/randomized-order.md | 58 +++++++++++++++++++++++++++++++++++++ 4 files changed, 85 insertions(+), 7 deletions(-) create mode 100644 docs/randomized-order.md diff --git a/README.md b/README.md index fb989bf..497f413 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,7 @@ A modern, feature-rich testing framework for Zig inspired by Jest, Vitest, and B - **Async Test Support** - Full async test execution with concurrent and sequential modes - **Timeout Handling** - Configurable timeouts at test, suite, and global levels with extension support - **Retries & Repeats** - Re-run failures, stress-run selected tests, and report flaky outcomes +- **Reproducible Randomization** - Shuffle suites and tests with replayable seeds - **Familiar API** - Describe/it syntax similar to Jest and Vitest - **Rich Assertions** - Comprehensive assertion library with `.expect()` and matchers - **Error Assertions** - `toThrow()` and `toThrowError()` for testing error handling @@ -102,6 +103,8 @@ This will automatically discover and run all `*.test.zig` files in the `tests` d - `--retry ` - Retry a failed test or discovered test file up to N times - `--repeat ` - Require N successful repetitions of every selected test - `--fail-on-flaky` - Return a failing exit status when a retry recovers a failure +- `--shuffle` - Randomize suites, tests, or discovered files with a generated seed +- `--seed ` - Replay a randomized order; supplying a seed implies `--shuffle` - `--shard-index ` - Run one one-based file shard - `--shard-count ` - Set the total number of file shards - `--no-color` - Disable color in child Zig test processes @@ -139,6 +142,10 @@ zig-test --test-dir tests --retry 2 --fail-on-flaky # Stress-run matching tests 100 times zig-test --test-dir tests --filter parser --repeat 100 +# Expose order dependencies, then replay the exact order with the printed seed +zig-test --test-dir tests --shuffle +zig-test --test-dir tests --seed 8675309 + # Run the second of four deterministic file shards zig-test --test-dir tests --shard-index 2 --shard-count 4 ``` @@ -185,7 +192,9 @@ precedence over configured values. "filter": "database", "retries": 2, "repeat": 10, - "fail_on_flaky": true + "fail_on_flaky": true, + "shuffle": true, + "seed": 8675309 }, "sharding": { "index": 1, @@ -204,6 +213,8 @@ limitations as their CLI equivalents; unsupported combinations fail explicitly. See [Retries, repeats, and flaky tests](docs/retries-and-repeats.md) for attempt semantics, exit codes, reporter output, and programmatic configuration. +See [Randomized test order](docs/randomized-order.md) for replay and ordering +guarantees. ### Parallel execution semantics @@ -212,7 +223,8 @@ and set `n_jobs`. A configured value is the exact maximum number of active test workers; omitting it uses the host's logical CPU count, with a one-worker fallback when detection is unavailable. -Suites and nested suites are visited in declaration order. `beforeAll` and +Suites and nested suites are visited in the selected order: declaration order +by default, or deterministic seed order when randomization is enabled. `beforeAll` and `afterAll` run once on the coordinating thread around that suite and its nested suites. Direct tests within a suite may overlap. Their inherited `beforeEach` and `afterEach` hooks run on the same worker as the test, so per-test hook order @@ -220,7 +232,7 @@ is preserved but hooks from different tests may overlap. Hook state and the allocator passed to tests must therefore be thread-safe. Reporter callbacks never run concurrently: test results are emitted in -declaration order after each parallel batch, making output and final totals +the selected order after each parallel batch, making output and final totals deterministic. A parallel batch is already scheduled as a unit, so `bail` cannot cancel tests that have started. @@ -665,6 +677,8 @@ zig-test --no-color | `--retry ` | `--retries ` | Retry each failed test up to N times | | `--repeat ` | | Require N successful runs of each selected test | | `--fail-on-flaky` | | Fail the run when a retry recovers a failure | +| `--shuffle` | | Randomize suite, test, and discovered-file order | +| `--seed ` | | Replay a randomized order; implies `--shuffle` | | `--verbose` | | Enable verbose output | | `--quiet` | `-q` | Minimal output | | `--no-color` | | Disable colored output | diff --git a/docs/execution-pipeline.md b/docs/execution-pipeline.md index 6b9de40..fbb797f 100644 --- a/docs/execution-pipeline.md +++ b/docs/execution-pipeline.md @@ -34,6 +34,9 @@ reporters, including the configured JUnit output path. child process. - Every logical result owns ordered attempt history. A failure followed by a successful retry is classified as `flaky`, distinct from an ordinary pass. +- Optional randomized execution uses one run seed across the selected plan. + The seed is carried by reporters and `TestResults`; see + [Randomized test order](randomized-order.md) for mode-specific guarantees. - Summaries always come from `TestResults`, rather than executor-local counters. ## Extension points @@ -53,10 +56,11 @@ model without coupling the execution API to JSON. Existing programmatic APIs remain source compatible: - `TestRunner.init`, `runTests`, and `runTestsWithOptions` are unchanged. -- `RunnerOptions` gains optional reporter, timeout, retry, repeat, and flaky-exit +- `RunnerOptions` gains optional reporter, timeout, retry, repeat, flaky-exit, + shuffle, and seed fields with backward-compatible defaults. - `runDiscoveredTests` and `LoaderOptions` remain available; optional reporter, - JUnit path, timeout, retry, repeat, flaky-exit, and writer fields default to + JUnit path, timeout, retry, repeat, flaky-exit, shuffle, seed, and writer fields default to the previous spec behavior. CLI hosts can inject a stdout writer with `reporter_writer`. Embedded callers diff --git a/docs/json-event-protocol.md b/docs/json-event-protocol.md index ac32b57..ed4b00d 100644 --- a/docs/json-event-protocol.md +++ b/docs/json-event-protocol.md @@ -35,8 +35,8 @@ events are emitted in attempt order immediately before their `test_end`. | Type | Data fields | | --- | --- | -| `run_start` | `total` | -| `run_end` | `total`, `passed`, `flaky`, `failed`, `skipped`, `duration_ns` | +| `run_start` | `total`, optional `random_seed` | +| `run_end` | `total`, `passed`, `flaky`, `failed`, `skipped`, `duration_ns`, optional `random_seed` | | `suite_start`, `suite_end` | `name` | | `test_start` | `name`, optional `suite` | | `test_end` | `name`, optional `suite`, `status`, `duration_ns`, optional `error_message` | @@ -51,6 +51,8 @@ Output streams are `stdout` or `stderr`. Optional fields are serialized as `null` when no value is available. Durations are unsigned integer nanoseconds. Percentages are numbers from 0 through 100. +When randomized ordering is enabled, both run events carry the reproducible +unsigned integer seed. ## Compatibility policy diff --git a/docs/randomized-order.md b/docs/randomized-order.md new file mode 100644 index 0000000..0a64d43 --- /dev/null +++ b/docs/randomized-order.md @@ -0,0 +1,58 @@ +# Randomized test order + +Randomized order is opt-in. Use `--shuffle` to generate a seed for the run: + +```bash +zig-test --shuffle +``` + +The selected reporter prints the generated seed at run start. If a test fails, +the human-readable reporters print a replay command as well. Pass that seed to +reproduce the order; `--seed` implies `--shuffle`: + +```bash +zig-test --seed 8675309 +``` + +Keep every other order-affecting option the same when replaying a run, including +the test set, filter, shard, execution mode, and suite structure. + +## Ordering guarantees + +Without `--shuffle` or `--seed`, execution retains declaration or discovery +order. + +In sequential programmatic execution, a seed deterministically orders root +suites, each suite's direct tests, and its nested suites. Lifecycle hooks keep +their normal scope: `beforeAll` and `afterAll` still wrap a suite, while +`beforeEach` and `afterEach` still wrap their selected test. + +In parallel programmatic execution, the seed determines suite traversal, each +worker queue, and serialized reporter output. Workers claim tests from that +queue in order, but operating-system scheduling means wall-clock start and +completion timing can differ between runs. Reporter order remains reproducible. + +In discovery mode, sharding selects files first and randomization orders the +files inside the selected shard. The seed does not change shard membership. +Retries and repetitions stay attached to their logical test or file and do not +trigger another shuffle. + +## Metadata + +The seed is stored as `random_seed` in JSON `run_start` and `run_end` events, +live UI events, and test-history files. JUnit output records it as a +`random_seed` property on each test suite. A non-randomized run reports a null +seed in JSON-compatible formats and omits the JUnit property. + +Programmatic callers can use `RunnerOptions.shuffle` and `RunnerOptions.seed`, +or the matching `LoaderOptions` fields. JSON configuration accepts the same +values under `test`: + +```json +{ + "test": { + "shuffle": true, + "seed": 8675309 + } +} +```