Add retries, repeated runs, and flaky-test reporting

CHANGELOG.md

@@ -16,6 +16,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+
+- Retry failed tests and discovered test files with complete attempt history.
+- Repeat mode for stress-running selected tests.
+- Flaky outcomes, optional fail-on-flaky exit behavior, and JSON/JUnit attempt data.
+
 ### Planned Features
 
 - Snapshot testing

README.md

@@ -8,6 +8,7 @@ A modern, feature-rich testing framework for Zig inspired by Jest, Vitest, and B
 - **Code Coverage** - Line, branch, and function coverage with HTML reports (via kcov/grindcov)
 - **Async Test Support** - Full async test execution with concurrent and sequential modes
 - **Timeout Handling** - Configurable timeouts at test, suite, and global levels with extension support
+- **Retries & Repeats** - Re-run failures, stress-run selected tests, and report flaky outcomes
 - **Familiar API** - Describe/it syntax similar to Jest and Vitest
 - **Rich Assertions** - Comprehensive assertion library with `.expect()` and matchers
 - **Error Assertions** - `toThrow()` and `toThrowError()` for testing error handling
@@ -98,6 +99,9 @@ This will automatically discover and run all `*.test.zig` files in the `tests` d
 - `--reporter <name>` - Use spec, dot, JSON, TAP, or JUnit output
 - `--junit-output <file>` - Set the JUnit XML path
 - `--timeout <ms>` - Mark files that exceed the global time budget as failed
+- `--retry <N>` - Retry a failed test or discovered test file up to N times
+- `--repeat <N>` - Require N successful repetitions of every selected test
+- `--fail-on-flaky` - Return a failing exit status when a retry recovers a failure
 - `--shard-index <N>` - Run one one-based file shard
 - `--shard-count <N>` - Set the total number of file shards
 - `--no-color` - Disable color in child Zig test processes
@@ -125,6 +129,12 @@ zig-test --test-dir tests --no-recursive
 # Stop on first failure
 zig-test --test-dir tests --bail
 
+# Retry failures twice and fail CI if any test is flaky
+zig-test --test-dir tests --retry 2 --fail-on-flaky
+
+# Stress-run matching tests 100 times
+zig-test --test-dir tests --filter parser --repeat 100
+
 # Run the second of four deterministic file shards
 zig-test --test-dir tests --shard-index 2 --shard-count 4
 ```
@@ -168,7 +178,10 @@ precedence over configured values.
     "test_dir": "tests",
     "pattern": "*.test.zig",
     "recursive": true,
-    "filter": "database"
+    "filter": "database",
+    "retries": 2,
+    "repeat": 10,
+    "fail_on_flaky": true
   },
   "sharding": {
     "index": 1,
@@ -185,6 +198,9 @@ The accepted top-level sections are `test`, `sharding`, `parallel`, `reporter`,
 `snapshot`, `watch`, `memory`, `ui`, and `coverage`. Options retain the same execution-mode
 limitations as their CLI equivalents; unsupported combinations fail explicitly.
 
+See [Retries, repeats, and flaky tests](docs/retries-and-repeats.md) for attempt
+semantics, exit codes, reporter output, and programmatic configuration.
+
 ### Parallel execution semantics
 
 Programmatic runners can enable bounded execution with `RunnerOptions.parallel`
@@ -642,6 +658,9 @@ zig-test --no-color
 | `--reporter <name>` | | Set reporter (spec, dot, json, tap, junit) |
 | `--junit-output <file>` | | Set the JUnit XML output path |
 | `--timeout <ms>` | | Set the global test timeout |
+| `--retry <N>` | `--retries <N>` | Retry each failed test up to N times |
+| `--repeat <N>` | | Require N successful runs of each selected test |
+| `--fail-on-flaky` | | Fail the run when a retry recovers a failure |
 | `--verbose` | | Enable verbose output |
 | `--quiet` | `-q` | Minimal output |
 | `--no-color` | | Disable colored output |
@@ -921,6 +940,7 @@ MIT License - see [LICENSE](LICENSE) for details.
 - [x] Memory profiling
 - [x] Strict JSON configuration files
 - [x] Parallel test execution
+- [x] Retries, repeated runs, and flaky-test reporting
 - [ ] Parameterized tests (it.each)
 - [ ] Property-based testing
 - [ ] IDE integration

build.zig

@@ -293,6 +293,21 @@ pub fn build(b: *std.Build) void {
     );
     test_step.dependOn(&run_discovery_tap.step);
 
+    const run_discovery_repeat = b.addRunArtifact(exe);
+    run_discovery_repeat.addArgs(&.{
+        "--test-dir",
+        "tests",
+        "--pattern",
+        "sample.test.zig",
+        "--retry",
+        "1",
+        "--repeat",
+        "2",
+        "--fail-on-flaky",
+        "--no-color",
+    });
+    test_step.dependOn(&run_discovery_repeat.step);
+
     const run_version = b.addRunArtifact(exe);
     run_version.addArg("--version");
     run_version.expectStdErrEqual(b.fmt("Zig Test Framework v{s}\n", .{package_version}));

docs/execution-pipeline.md

@@ -26,10 +26,14 @@ reporters, including the configured JUnit output path.
 - Registered tests match filters directly by test name. Discovery forwards the
   same filter to `zig test`, where Zig applies it to names inside each file.
 - Bail stops scheduling after the shared result model records a failure. Work
-  already started by a parallel batch cannot be cancelled.
+  already started by a parallel batch cannot be cancelled. With retries, bail
+  is evaluated only after the retry budget is exhausted.
 - A global timeout compares elapsed execution time with the same policy in both
-  executors. It marks an over-budget result failed after control returns; it
-  does not yet terminate an in-process function or external child process.
+  executors on every attempt. It marks an over-budget attempt failed after
+  control returns; it does not yet terminate an in-process function or external
+  child process.
+- Every logical result owns ordered attempt history. A failure followed by a
+  successful retry is classified as `flaky`, distinct from an ordinary pass.
 - Summaries always come from `TestResults`, rather than executor-local counters.
 
 ## Extension points
@@ -48,10 +52,11 @@ depend on one serialization format.
 Existing programmatic APIs remain source compatible:
 
 - `TestRunner.init`, `runTests`, and `runTestsWithOptions` are unchanged.
-- `RunnerOptions` only gains optional `junit_output`, `timeout_ms`, and
-  `reporter_writer` fields.
+- `RunnerOptions` gains optional reporter, timeout, retry, repeat, and flaky-exit
+  fields with backward-compatible defaults.
 - `runDiscoveredTests` and `LoaderOptions` remain available; optional reporter,
-  JUnit path, timeout, and writer fields default to the previous spec behavior.
+  JUnit path, timeout, retry, repeat, flaky-exit, and writer fields default to
+  the previous spec behavior.
 
 CLI hosts can inject a stdout writer with `reporter_writer`. Embedded callers
 may leave it unset; reporter output then uses stderr so it cannot corrupt Zig's

docs/retries-and-repeats.md

@@ -0,0 +1,92 @@
+# Retries, repeats, and flaky tests
+
+The runner records each execution of a logical test as an attempt. An attempt
+contains its one-based number, repetition number, status, duration, and error
+message. Registered tests and discovered test files use the same model.
+
+## Command-line use
+
+```bash
+# One initial attempt plus as many as two retries after failure
+zig-test --test-dir tests --retry 2
+
+# Require every selected test to pass 100 times
+zig-test --test-dir tests --filter parser --repeat 100
+
+# Make a recovered failure fail the overall run in CI
+zig-test --test-dir tests --retry 2 --fail-on-flaky
+```
+
+`--retries` is an alias for `--retry`. A retry count is the number of
+*additional* attempts after a failure. `--repeat N` requires N successful
+repetitions; each repetition receives its own retry budget. Attempt history is
+ordered across all repetitions.
+
+Discovery operates at test-file granularity because each file is an external
+`zig test` process. Programmatic mode operates at individual-test granularity.
+
+## Outcomes and exit status
+
+- `passed`: every attempt passed without a retry.
+- `flaky`: at least one attempt failed, but a retry recovered and all required
+  repetitions eventually passed.
+- `failed`: one repetition exhausted its retry budget.
+- `skipped`: the test was not selected for execution.
+
+Failed tests always make the process exit unsuccessfully. Flaky tests succeed
+by default so retries can recover an intermittent failure. `--fail-on-flaky`
+changes that policy and returns an unsuccessful exit status when any test is
+flaky.
+
+`--bail` is evaluated after retries, so one failed attempt does not stop the
+run. The runner stops scheduling only after a logical test exhausts its retry
+budget. The timeout is evaluated independently for every attempt; a timed-out
+attempt may be retried.
+
+When retries or repeats are configured for registered tests, execution is
+sequential even if parallel mode was requested. This preserves deterministic
+attempt ordering and hook behavior.
+
+## Programmatic configuration
+
+Set defaults for a registered-test run with `RunnerOptions`:
+
+```zig
+const options = ztf.RunnerOptions{
+    .retries = 2,
+    .repeat = 10,
+    .fail_on_flaky = true,
+};
+const passed = try ztf.runTestsWithOptions(allocator, registry, options);
+```
+
+An individual registered test can override the run-level retry count:
+
+```zig
+try suite.addTest(
+    ztf.TestCase.init("eventually consistent", testEventuallyConsistent)
+        .withRetries(5),
+);
+```
+
+`LoaderOptions` exposes the same `retries`, `repeat`, and `fail_on_flaky`
+fields for callers that run discovered files directly. Strict JSON
+configuration uses the corresponding fields in the `test` object:
+
+```json
+{
+  "test": {
+    "retries": 2,
+    "repeat": 10,
+    "fail_on_flaky": true
+  }
+}
+```
+
+## Reporter data
+
+The spec and dot reporters show flaky results separately from passes. JSON test
+objects include an `attempts` array with `number`, `repetition`, `status`, and
+`durationNs` fields plus an optional `error`. The JSON summary includes a
+`flaky` count. JUnit output includes the flaky count on `<testsuites>` and emits
+ordered attempt data in each retried test case's `<system-out>` element.

src/cli.zig

@@ -45,6 +45,10 @@ pub const CLIOptions = struct {
     junit_output: ?[]const u8 = null,
     // Timeout options
     timeout: ?u64 = null, // Global timeout in milliseconds
+    // Retry and repetition options
+    retries: usize = 0,
+    repeat: usize = 1,
+    fail_on_flaky: bool = false,
 };
 
 pub const version = build_options.version;
@@ -75,6 +79,9 @@ pub const CLI = struct {
         self.options.no_recursive = !config.test_options.recursive;
         self.options.filter = config.test_options.filter;
         self.options.timeout = config.test_options.timeout;
+        self.options.retries = config.test_options.retries;
+        self.options.repeat = config.test_options.repeat;
+        self.options.fail_on_flaky = config.test_options.fail_on_flaky;
         self.options.shard_index = config.sharding.index;
         self.options.shard_count = config.sharding.count;
         self.options.parallel = config.parallel.enabled;
@@ -107,6 +114,32 @@ pub const CLI = struct {
                 self.options.version = true;
             } else if (std.mem.eql(u8, arg, "--bail") or std.mem.eql(u8, arg, "-b")) {
                 self.options.bail = true;
+            } else if (std.mem.eql(u8, arg, "--retry") or std.mem.eql(u8, arg, "--retries")) {
+                if (i + 1 >= args.len) {
+                    std.debug.print("Error: {s} requires a value\n", .{arg});
+                    return CLIError.MissingValue;
+                }
+                i += 1;
+                self.options.retries = std.fmt.parseInt(usize, args[i], 10) catch {
+                    std.debug.print("Error: {s} must be a valid number\n", .{arg});
+                    return CLIError.InvalidArgument;
+                };
+            } else if (std.mem.eql(u8, arg, "--repeat")) {
+                if (i + 1 >= args.len) {
+                    std.debug.print("Error: --repeat requires a value\n", .{});
+                    return CLIError.MissingValue;
+                }
+                i += 1;
+                self.options.repeat = std.fmt.parseInt(usize, args[i], 10) catch {
+                    std.debug.print("Error: --repeat must be a valid positive number\n", .{});
+                    return CLIError.InvalidArgument;
+                };
+                if (self.options.repeat == 0) {
+                    std.debug.print("Error: --repeat must be greater than zero\n", .{});
+                    return CLIError.InvalidArgument;
+                }
+            } else if (std.mem.eql(u8, arg, "--fail-on-flaky")) {
+                self.options.fail_on_flaky = true;
             } else if (std.mem.eql(u8, arg, "--verbose")) {
                 self.options.verbose = true;
             } else if (std.mem.eql(u8, arg, "--quiet") or std.mem.eql(u8, arg, "-q")) {
@@ -351,6 +384,9 @@ pub const CLI = struct {
             \\    -h, --help              Show this help message
             \\    -v, --version           Show version information
             \\    -b, --bail              Stop test execution on first failure
+            \\    --retry <N>             Retry each failed test up to N times
+            \\    --repeat <N>            Require N successful runs of each selected test
+            \\    --fail-on-flaky         Exit unsuccessfully when a retry recovers a failure
             \\    --filter <pattern>      Run only tests matching pattern
             \\    --grep <pattern>        Same as --filter (alias)
             \\    --reporter <name>       Set reporter type (spec, dot, json, tap, junit)
@@ -414,6 +450,8 @@ pub const CLI = struct {
             \\    zig-test --update-snapshots           Update all snapshots
             \\    zig-test --watch                      Run in watch mode
             \\    zig-test --timeout 10000              Set 10s timeout for all tests
+            \\    zig-test --retry 2 --fail-on-flaky    Detect and reject flaky tests
+            \\    zig-test --repeat 100 --filter parser Stress-run selected tests
             \\    zig-test --profile-memory --fail-on-leak
             \\    zig-test --parallel --jobs 4          Run with 4 parallel workers
             \\    zig-test --config zig-test.json       Load config from file
@@ -441,6 +479,9 @@ pub const CLI = struct {
             .n_jobs = self.options.jobs,
             .junit_output = self.options.junit_output orelse "test-results.xml",
             .timeout_ms = self.options.timeout,
+            .retries = self.options.retries,
+            .repeat = self.options.repeat,
+            .fail_on_flaky = self.options.fail_on_flaky,
         };
     }
 };
@@ -662,6 +703,35 @@ test "CLI parse timeout" {
     try std.testing.expectEqual(@as(?u64, 5000), cli.options.timeout);
 }
 
+test "CLI parses retry repeat and flaky exit policy" {
+    var cli = CLI.init(std.testing.allocator);
+    const args = [_][]const u8{
+        "zig-test",
+        "--retry",
+        "2",
+        "--repeat",
+        "5",
+        "--fail-on-flaky",
+    };
+    try cli.parse(&args);
+    try std.testing.expectEqual(@as(usize, 2), cli.options.retries);
+    try std.testing.expectEqual(@as(usize, 5), cli.options.repeat);
+    try std.testing.expect(cli.options.fail_on_flaky);
+
+    const runner_options = cli.toRunnerOptions();
+    try std.testing.expectEqual(@as(usize, 2), runner_options.retries);
+    try std.testing.expectEqual(@as(usize, 5), runner_options.repeat);
+    try std.testing.expect(runner_options.fail_on_flaky);
+}
+
+test "CLI rejects a zero repeat count" {
+    var cli = CLI.init(std.testing.allocator);
+    try std.testing.expectError(
+        CLIError.InvalidArgument,
+        cli.parse(&.{ "zig-test", "--repeat", "0" }),
+    );
+}
+
 test "CLI rejects invalid coverage tool" {
     var cli = CLI.init(std.testing.allocator);
     const args = [_][]const u8{ "zig-test", "--coverage-tool", "unknown" };