Add reproducible randomized test order

README.md

@@ -9,6 +9,7 @@ A modern, feature-rich testing framework for Zig inspired by Jest, Vitest, and B
 - **Async Test Support** - Full async test execution with concurrent and sequential modes
 - **Timeout Handling** - Configurable timeouts at test, suite, and global levels with extension support
 - **Retries & Repeats** - Re-run failures, stress-run selected tests, and report flaky outcomes
+- **Reproducible Randomization** - Shuffle suites and tests with replayable seeds
 - **Familiar API** - Describe/it syntax similar to Jest and Vitest
 - **Rich Assertions** - Comprehensive assertion library with `.expect()` and matchers
 - **Error Assertions** - `toThrow()` and `toThrowError()` for testing error handling
@@ -102,6 +103,8 @@ This will automatically discover and run all `*.test.zig` files in the `tests` d
 - `--retry <N>` - Retry a failed test or discovered test file up to N times
 - `--repeat <N>` - Require N successful repetitions of every selected test
 - `--fail-on-flaky` - Return a failing exit status when a retry recovers a failure
+- `--shuffle` - Randomize suites, tests, or discovered files with a generated seed
+- `--seed <N>` - Replay a randomized order; supplying a seed implies `--shuffle`
 - `--shard-index <N>` - Run one one-based file shard
 - `--shard-count <N>` - Set the total number of file shards
 - `--no-color` - Disable color in child Zig test processes
@@ -139,6 +142,10 @@ zig-test --test-dir tests --retry 2 --fail-on-flaky
 # Stress-run matching tests 100 times
 zig-test --test-dir tests --filter parser --repeat 100
 
+# Expose order dependencies, then replay the exact order with the printed seed
+zig-test --test-dir tests --shuffle
+zig-test --test-dir tests --seed 8675309
+
 # Run the second of four deterministic file shards
 zig-test --test-dir tests --shard-index 2 --shard-count 4
 ```
@@ -185,7 +192,9 @@ precedence over configured values.
     "filter": "database",
     "retries": 2,
     "repeat": 10,
-    "fail_on_flaky": true
+    "fail_on_flaky": true,
+    "shuffle": true,
+    "seed": 8675309
   },
   "sharding": {
     "index": 1,
@@ -204,6 +213,8 @@ limitations as their CLI equivalents; unsupported combinations fail explicitly.
 
 See [Retries, repeats, and flaky tests](docs/retries-and-repeats.md) for attempt
 semantics, exit codes, reporter output, and programmatic configuration.
+See [Randomized test order](docs/randomized-order.md) for replay and ordering
+guarantees.
 
 ### Parallel execution semantics
 
@@ -212,15 +223,16 @@ and set `n_jobs`. A configured value is the exact maximum number of active test
 workers; omitting it uses the host's logical CPU count, with a one-worker
 fallback when detection is unavailable.
 
-Suites and nested suites are visited in declaration order. `beforeAll` and
+Suites and nested suites are visited in the selected order: declaration order
+by default, or deterministic seed order when randomization is enabled. `beforeAll` and
 `afterAll` run once on the coordinating thread around that suite and its nested
 suites. Direct tests within a suite may overlap. Their inherited `beforeEach`
 and `afterEach` hooks run on the same worker as the test, so per-test hook order
 is preserved but hooks from different tests may overlap. Hook state and the
 allocator passed to tests must therefore be thread-safe.
 
 Reporter callbacks never run concurrently: test results are emitted in
-declaration order after each parallel batch, making output and final totals
+the selected order after each parallel batch, making output and final totals
 deterministic. A parallel batch is already scheduled as a unit, so `bail` cannot
 cancel tests that have started.
 
@@ -665,6 +677,8 @@ zig-test --no-color
 | `--retry <N>` | `--retries <N>` | Retry each failed test up to N times |
 | `--repeat <N>` | | Require N successful runs of each selected test |
 | `--fail-on-flaky` | | Fail the run when a retry recovers a failure |
+| `--shuffle` | | Randomize suite, test, and discovered-file order |
+| `--seed <N>` | | Replay a randomized order; implies `--shuffle` |
 | `--verbose` | | Enable verbose output |
 | `--quiet` | `-q` | Minimal output |
 | `--no-color` | | Disable colored output |

docs/execution-pipeline.md

@@ -34,6 +34,9 @@ reporters, including the configured JUnit output path.
   child process.
 - Every logical result owns ordered attempt history. A failure followed by a
   successful retry is classified as `flaky`, distinct from an ordinary pass.
+- Optional randomized execution uses one run seed across the selected plan.
+  The seed is carried by reporters and `TestResults`; see
+  [Randomized test order](randomized-order.md) for mode-specific guarantees.
 - Summaries always come from `TestResults`, rather than executor-local counters.
 
 ## Extension points
@@ -53,10 +56,11 @@ model without coupling the execution API to JSON.
 Existing programmatic APIs remain source compatible:
 
 - `TestRunner.init`, `runTests`, and `runTestsWithOptions` are unchanged.
-- `RunnerOptions` gains optional reporter, timeout, retry, repeat, and flaky-exit
+- `RunnerOptions` gains optional reporter, timeout, retry, repeat, flaky-exit,
+  shuffle, and seed
   fields with backward-compatible defaults.
 - `runDiscoveredTests` and `LoaderOptions` remain available; optional reporter,
-  JUnit path, timeout, retry, repeat, flaky-exit, and writer fields default to
+  JUnit path, timeout, retry, repeat, flaky-exit, shuffle, seed, and writer fields default to
   the previous spec behavior.
 
 CLI hosts can inject a stdout writer with `reporter_writer`. Embedded callers

docs/json-event-protocol.md

@@ -35,8 +35,8 @@ events are emitted in attempt order immediately before their `test_end`.
 
 | Type | Data fields |
 | --- | --- |
-| `run_start` | `total` |
-| `run_end` | `total`, `passed`, `flaky`, `failed`, `skipped`, `duration_ns` |
+| `run_start` | `total`, optional `random_seed` |
+| `run_end` | `total`, `passed`, `flaky`, `failed`, `skipped`, `duration_ns`, optional `random_seed` |
 | `suite_start`, `suite_end` | `name` |
 | `test_start` | `name`, optional `suite` |
 | `test_end` | `name`, optional `suite`, `status`, `duration_ns`, optional `error_message` |
@@ -51,6 +51,8 @@ Output streams are `stdout` or `stderr`.
 
 Optional fields are serialized as `null` when no value is available. Durations
 are unsigned integer nanoseconds. Percentages are numbers from 0 through 100.
+When randomized ordering is enabled, both run events carry the reproducible
+unsigned integer seed.
 
 ## Compatibility policy
 

docs/randomized-order.md

@@ -0,0 +1,58 @@
+# Randomized test order
+
+Randomized order is opt-in. Use `--shuffle` to generate a seed for the run:
+
+```bash
+zig-test --shuffle
+```
+
+The selected reporter prints the generated seed at run start. If a test fails,
+the human-readable reporters print a replay command as well. Pass that seed to
+reproduce the order; `--seed` implies `--shuffle`:
+
+```bash
+zig-test --seed 8675309
+```
+
+Keep every other order-affecting option the same when replaying a run, including
+the test set, filter, shard, execution mode, and suite structure.
+
+## Ordering guarantees
+
+Without `--shuffle` or `--seed`, execution retains declaration or discovery
+order.
+
+In sequential programmatic execution, a seed deterministically orders root
+suites, each suite's direct tests, and its nested suites. Lifecycle hooks keep
+their normal scope: `beforeAll` and `afterAll` still wrap a suite, while
+`beforeEach` and `afterEach` still wrap their selected test.
+
+In parallel programmatic execution, the seed determines suite traversal, each
+worker queue, and serialized reporter output. Workers claim tests from that
+queue in order, but operating-system scheduling means wall-clock start and
+completion timing can differ between runs. Reporter order remains reproducible.
+
+In discovery mode, sharding selects files first and randomization orders the
+files inside the selected shard. The seed does not change shard membership.
+Retries and repetitions stay attached to their logical test or file and do not
+trigger another shuffle.
+
+## Metadata
+
+The seed is stored as `random_seed` in JSON `run_start` and `run_end` events,
+live UI events, and test-history files. JUnit output records it as a
+`random_seed` property on each test suite. A non-randomized run reports a null
+seed in JSON-compatible formats and omits the JUnit property.
+
+Programmatic callers can use `RunnerOptions.shuffle` and `RunnerOptions.seed`,
+or the matching `LoaderOptions` fields. JSON configuration accepts the same
+values under `test`:
+
+```json
+{
+  "test": {
+    "shuffle": true,
+    "seed": 8675309
+  }
+}
+```

src/cli.zig

@@ -49,6 +49,8 @@ pub const CLIOptions = struct {
     retries: usize = 0,
     repeat: usize = 1,
     fail_on_flaky: bool = false,
+    shuffle: bool = false,
+    seed: ?u64 = null,
 };
 
 pub const version = build_options.version;
@@ -82,6 +84,8 @@ pub const CLI = struct {
         self.options.retries = config.test_options.retries;
         self.options.repeat = config.test_options.repeat;
         self.options.fail_on_flaky = config.test_options.fail_on_flaky;
+        self.options.shuffle = config.test_options.shuffle or config.test_options.seed != null;
+        self.options.seed = config.test_options.seed;
         self.options.shard_index = config.sharding.index;
         self.options.shard_count = config.sharding.count;
         self.options.parallel = config.parallel.enabled;
@@ -140,6 +144,19 @@ pub const CLI = struct {
                 }
             } else if (std.mem.eql(u8, arg, "--fail-on-flaky")) {
                 self.options.fail_on_flaky = true;
+            } else if (std.mem.eql(u8, arg, "--shuffle")) {
+                self.options.shuffle = true;
+            } else if (std.mem.eql(u8, arg, "--seed")) {
+                if (i + 1 >= args.len) {
+                    std.debug.print("Error: --seed requires a value\n", .{});
+                    return CLIError.MissingValue;
+                }
+                i += 1;
+                self.options.seed = std.fmt.parseInt(u64, args[i], 10) catch {
+                    std.debug.print("Error: --seed must be a valid unsigned integer\n", .{});
+                    return CLIError.InvalidArgument;
+                };
+                self.options.shuffle = true;
             } else if (std.mem.eql(u8, arg, "--verbose")) {
                 self.options.verbose = true;
             } else if (std.mem.eql(u8, arg, "--quiet") or std.mem.eql(u8, arg, "-q")) {
@@ -386,6 +403,8 @@ pub const CLI = struct {
             \\    --retry <N>             Retry each failed test up to N times
             \\    --repeat <N>            Require N successful runs of each selected test
             \\    --fail-on-flaky         Exit unsuccessfully when a retry recovers a failure
+            \\    --shuffle               Randomize suite and test order
+            \\    --seed <N>              Reproduce a randomized order (implies --shuffle)
             \\    --filter <pattern>      Run only tests matching pattern
             \\    --grep <pattern>        Same as --filter (alias)
             \\    --reporter <name>       Set reporter type (spec, dot, json, tap, junit)
@@ -452,6 +471,8 @@ pub const CLI = struct {
             \\    zig-test --timeout 10000              Set 10s timeout for all tests
             \\    zig-test --retry 2 --fail-on-flaky    Detect and reject flaky tests
             \\    zig-test --repeat 100 --filter parser Stress-run selected tests
+            \\    zig-test --shuffle                  Run in a generated random order
+            \\    zig-test --seed 12345               Replay a randomized order
             \\    zig-test --profile-memory --fail-on-leak
             \\    zig-test --parallel --jobs 4          Run with 4 parallel workers
             \\    zig-test --config zig-test.json       Load config from file
@@ -482,6 +503,8 @@ pub const CLI = struct {
             .retries = self.options.retries,
             .repeat = self.options.repeat,
             .fail_on_flaky = self.options.fail_on_flaky,
+            .shuffle = self.options.shuffle,
+            .seed = self.options.seed,
         };
     }
 };
@@ -724,6 +747,17 @@ test "CLI parses retry repeat and flaky exit policy" {
     try std.testing.expect(runner_options.fail_on_flaky);
 }
 
+test "CLI parses shuffle and a seed implies it" {
+    var cli = CLI.init(std.testing.allocator);
+    try cli.parse(&.{ "zig-test", "--seed", "12345" });
+    try std.testing.expect(cli.options.shuffle);
+    try std.testing.expectEqual(@as(?u64, 12345), cli.options.seed);
+
+    const runner_options = cli.toRunnerOptions();
+    try std.testing.expect(runner_options.shuffle);
+    try std.testing.expectEqual(@as(?u64, 12345), runner_options.seed);
+}
+
 test "CLI rejects a zero repeat count" {
     var cli = CLI.init(std.testing.allocator);
     try std.testing.expectError(

src/config.zig

@@ -32,6 +32,8 @@ pub const TestOptions = struct {
     retries: usize = 0,
     repeat: usize = 1,
     fail_on_flaky: bool = false,
+    shuffle: bool = false,
+    seed: ?u64 = null,
 };
 
 pub const ParallelOptions = struct {
@@ -207,6 +209,8 @@ test "TestConfig defaults match CLI discovery defaults" {
     try std.testing.expectEqual(@as(usize, 0), config.test_options.retries);
     try std.testing.expectEqual(@as(usize, 1), config.test_options.repeat);
     try std.testing.expect(!config.test_options.fail_on_flaky);
+    try std.testing.expect(!config.test_options.shuffle);
+    try std.testing.expect(config.test_options.seed == null);
     try std.testing.expect(config.sharding.index == null);
     try std.testing.expect(config.sharding.count == null);
     try std.testing.expect(!config.parallel.enabled);
@@ -247,6 +251,8 @@ test "ConfigLoader loads supported JSON and rejects unknown keys" {
     try std.testing.expectEqual(@as(usize, 1), config.test_options.retries);
     try std.testing.expectEqual(@as(usize, 2), config.test_options.repeat);
     try std.testing.expect(!config.test_options.fail_on_flaky);
+    try std.testing.expect(config.test_options.shuffle);
+    try std.testing.expectEqual(@as(?u64, 42), config.test_options.seed);
     try std.testing.expectEqual(@as(?usize, 1), config.sharding.index);
     try std.testing.expectEqual(@as(?usize, 1), config.sharding.count);
 

src/event_protocol.zig

@@ -7,14 +7,18 @@ pub const Status = enum { pending, running, passed, flaky, failed, skipped };
 pub const HookKind = enum { before_all, before_each, after_each, after_all };
 pub const OutputStream = enum { stdout, stderr };
 
-pub const RunStart = struct { total: usize };
+pub const RunStart = struct {
+    total: usize,
+    random_seed: ?u64 = null,
+};
 pub const RunEnd = struct {
     total: usize,
     passed: usize,
     flaky: usize,
     failed: usize,
     skipped: usize,
     duration_ns: u64 = 0,
+    random_seed: ?u64 = null,
 };
 pub const Suite = struct { name: []const u8 };
 pub const TestStart = struct { name: []const u8, suite: ?[]const u8 = null };

src/lib.zig

@@ -27,6 +27,7 @@ pub const timeout = @import("timeout.zig");
 pub const progress = @import("progress.zig");
 pub const time = @import("time.zig");
 pub const compat = @import("compat.zig");
+pub const random_order = @import("random_order.zig");
 
 // Re-export commonly used types and functions
 pub const expect = assertions.expect;

src/main.zig

@@ -189,6 +189,8 @@ pub fn main(init: std.process.Init.Minimal) !void {
             .retries = cli_parser.options.retries,
             .repeat = cli_parser.options.repeat,
             .fail_on_flaky = cli_parser.options.fail_on_flaky,
+            .shuffle = cli_parser.options.shuffle,
+            .seed = cli_parser.options.seed,
             .reporter_writer = &stdout_writer.interface,
             .coverage_options = cov_opts,
             .ui_server = if (ui_server) |*server| server else null,
@@ -235,6 +237,8 @@ pub fn main(init: std.process.Init.Minimal) !void {
             .retries = cli_parser.options.retries,
             .repeat = cli_parser.options.repeat,
             .fail_on_flaky = cli_parser.options.fail_on_flaky,
+            .shuffle = cli_parser.options.shuffle,
+            .seed = cli_parser.options.seed,
             .reporter_writer = &stdout_writer.interface,
             .coverage_options = cov_opts,
             .ui_server = if (ui_server) |*server| server else null,