Unify discovery and programmatic test execution

README.md

@@ -95,15 +95,20 @@ This will automatically discover and run all `*.test.zig` files in the `tests` d
 - `--no-recursive` - Disable recursive directory search
 - `--bail` - Stop on first failure
 - `--filter <text>` / `--grep <text>` - Run only Zig tests whose names contain the text
+- `--reporter <name>` - Use spec, dot, JSON, TAP, or JUnit output
+- `--junit-output <file>` - Set the JUnit XML path
+- `--timeout <ms>` - Mark files that exceed the global time budget as failed
 - `--shard-index <N>` - Run one one-based file shard
 - `--shard-count <N>` - Set the total number of file shards
 - `--no-color` - Disable color in child Zig test processes
 - `--verbose` - Show detailed output
 
-Discovery mode rejects options that it cannot honor yet instead of silently
-ignoring them. Reporter selection, quiet output, timeouts, parallel jobs,
-snapshot updates, memory profiling, JUnit output, and the web UI require the
-shared execution pipeline planned for future releases.
+Discovery and programmatic execution share the same test-plan, lifecycle,
+result, and reporter model. Discovery still rejects options it cannot honor
+instead of silently ignoring them: quiet output, parallel jobs, snapshot
+updates, memory profiling, and the web UI remain programmatic-only. See
+[the execution pipeline architecture](docs/execution-pipeline.md) for extension
+points, timeout semantics, and migration guidance.
 
 **Examples:**
 
@@ -634,7 +639,9 @@ zig-test --no-color
 | `--version` | `-v` | Show version information |
 | `--bail` | `-b` | Stop on first failure |
 | `--filter <pattern>` | | Run only tests matching pattern |
-| `--reporter <name>` | | Set reporter (spec, dot, json) |
+| `--reporter <name>` | | Set reporter (spec, dot, json, tap, junit) |
+| `--junit-output <file>` | | Set the JUnit XML output path |
+| `--timeout <ms>` | | Set the global test timeout |
 | `--verbose` | | Enable verbose output |
 | `--quiet` | `-q` | Minimal output |
 | `--no-color` | | Disable colored output |

build.zig

@@ -286,6 +286,13 @@ pub fn build(b: *std.Build) void {
     run_shard_two.addArgs(&.{ "--test-dir", "tests", "--pattern", "sample.test.zig", "--shard-index", "2", "--shard-count", "2", "--no-color" });
     test_step.dependOn(&run_shard_two.step);
 
+    const run_discovery_tap = b.addRunArtifact(exe);
+    run_discovery_tap.addArgs(&.{ "--test-dir", "tests", "--pattern", "sample.test.zig", "--reporter", "tap", "--no-color" });
+    run_discovery_tap.expectStdOutEqual(
+        "TAP version 14\n1..1\n# Subtest: sample.test.zig\nok 1 - sample.test.zig\n",
+    );
+    test_step.dependOn(&run_discovery_tap.step);
+
     const run_version = b.addRunArtifact(exe);
     run_version.addArg("--version");
     run_version.expectStdErrEqual(b.fmt("Zig Test Framework v{s}\n", .{package_version}));

docs/execution-pipeline.md

@@ -0,0 +1,62 @@
+# Unified execution pipeline
+
+The framework has two plan producers and two executor implementations, but one
+execution contract.
+
+1. Programmatic registration or file discovery produces a `TestPlan`.
+2. A registered-test or external-process executor consumes that plan.
+3. The executor emits typed `LifecycleEvent` values through `EventStream`.
+4. A selected reporter consumes the events and a shared `TestResults` summary.
+
+`PlanItem.registered` retains the suite and test-case pointers needed for hooks
+and in-process execution. `PlanItem.discovered` retains the discovered file used
+by the external `zig test` executor. `PlanItem.identity()` gives extensions a
+common stable identity without erasing executor-specific information.
+
+## Shared lifecycle and results
+
+Both executors emit run, suite, and test start/end events. Discovered files are
+represented as one synthetic suite and test result per external process. This
+means spec, dot, JSON, TAP, and JUnit reporters receive the same lifecycle and
+summary model in either mode. `ReporterSet` is the single factory for built-in
+reporters, including the configured JUnit output path.
+
+`ExecutionPolicy` owns common filtering, bail, and timeout decisions:
+
+- Registered tests match filters directly by test name. Discovery forwards the
+  same filter to `zig test`, where Zig applies it to names inside each file.
+- Bail stops scheduling after the shared result model records a failure. Work
+  already started by a parallel batch cannot be cancelled.
+- A global timeout compares elapsed execution time with the same policy in both
+  executors. It marks an over-budget result failed after control returns; it
+  does not yet terminate an in-process function or external child process.
+- Summaries always come from `TestResults`, rather than executor-local counters.
+
+## Extension points
+
+New plan producers should append `PlanItem` values. New executors should emit
+`LifecycleEvent` values and update `TestResults`. New reporters can continue to
+implement `Reporter.VTable`; `EventStream` adapts the typed lifecycle to that
+stable callback interface.
+
+The event types are intentionally internal Zig structures. The separately
+versioned JSON wire protocol is built on this layer rather than making this API
+depend on one serialization format.
+
+## Migration
+
+Existing programmatic APIs remain source compatible:
+
+- `TestRunner.init`, `runTests`, and `runTestsWithOptions` are unchanged.
+- `RunnerOptions` only gains optional `junit_output`, `timeout_ms`, and
+  `reporter_writer` fields.
+- `runDiscoveredTests` and `LoaderOptions` remain available; optional reporter,
+  JUnit path, timeout, and writer fields default to the previous spec behavior.
+
+CLI hosts can inject a stdout writer with `reporter_writer`. Embedded callers
+may leave it unset; reporter output then uses stderr so it cannot corrupt Zig's
+stdout-based test protocol.
+
+Code that implements a custom execution path can migrate incrementally by first
+building a `TestPlan`, then routing callbacks through `EventStream`. Existing
+custom reporters require no migration.

src/cli.zig

@@ -326,7 +326,6 @@ pub const CLI = struct {
         if (!self.options.watch and self.options.watch_debounce != 300) return "--watch-debounce";
         if (!self.options.ui and self.options.ui_port != 8080) return "--ui-port";
         if (!self.options.ui and !std.mem.eql(u8, self.options.ui_host, "127.0.0.1")) return "--ui-host";
-        if (self.options.reporter != .spec) return "--reporter";
         if (self.options.quiet) return "--quiet";
         if (self.options.parallel) return "--parallel";
         if (self.options.jobs != null) return "--jobs";
@@ -335,8 +334,6 @@ pub const CLI = struct {
         if (self.options.profile_memory) return "--profile-memory";
         if (self.options.memory_threshold != 0) return "--memory-threshold";
         if (self.options.fail_on_leak) return "--fail-on-leak";
-        if (self.options.junit_output != null) return "--junit-output";
-        if (self.options.timeout != null) return "--timeout";
         if (self.options.ui) return "--ui";
         return null;
     }
@@ -442,6 +439,8 @@ pub const CLI = struct {
             .use_colors = !self.options.no_color,
             .parallel = self.options.parallel,
             .n_jobs = self.options.jobs,
+            .junit_output = self.options.junit_output orelse "test-results.xml",
+            .timeout_ms = self.options.timeout,
         };
     }
 };
@@ -691,18 +690,18 @@ test "discovery accepts implemented options" {
     try std.testing.expectEqual(@as(?[]const u8, null), cli.unsupportedDiscoveryOption());
 }
 
-test "discovery identifies unsupported options" {
+test "discovery accepts reporter selection" {
     var cli = CLI.init(std.testing.allocator);
     const args = [_][]const u8{ "zig-test", "--test-dir", "tests", "--reporter", "json" };
     try cli.parse(&args);
-    try std.testing.expectEqualStrings("--reporter", cli.unsupportedDiscoveryOption().?);
+    try std.testing.expectEqual(@as(?[]const u8, null), cli.unsupportedDiscoveryOption());
 }
 
-test "discovery reports timeout and parallel options as unsupported" {
+test "discovery accepts timeout and reports parallel as unsupported" {
     var cli = CLI.init(std.testing.allocator);
 
     cli.options.timeout = 1000;
-    try std.testing.expectEqualStrings("--timeout", cli.unsupportedDiscoveryOption().?);
+    try std.testing.expectEqual(@as(?[]const u8, null), cli.unsupportedDiscoveryOption());
 
     cli.options.timeout = null;
     cli.options.parallel = true;

src/lib.zig

@@ -5,6 +5,7 @@ pub const assertions = @import("assertions.zig");
 pub const suite = @import("suite.zig");
 pub const test_runner = @import("test_runner.zig");
 pub const reporter = @import("reporter.zig");
+pub const pipeline = @import("pipeline.zig");
 pub const matchers = @import("matchers.zig");
 pub const mock = @import("mock.zig");
 pub const cli = @import("cli.zig");
@@ -79,6 +80,15 @@ pub const JsonReporter = reporter.JsonReporter;
 pub const TAPReporter = reporter.TAPReporter;
 pub const JUnitReporter = reporter.JUnitReporter;
 pub const Colors = reporter.Colors;
+pub const ReporterSet = reporter.ReporterSet;
+
+// Unified execution pipeline
+pub const TestPlan = pipeline.TestPlan;
+pub const PlanItem = pipeline.PlanItem;
+pub const PlanOrigin = pipeline.PlanOrigin;
+pub const ExecutionPolicy = pipeline.ExecutionPolicy;
+pub const LifecycleEvent = pipeline.LifecycleEvent;
+pub const EventStream = pipeline.EventStream;
 
 // Matchers
 pub const Matchers = matchers.Matchers;

src/main.zig

@@ -161,6 +161,11 @@ pub fn main(init: std.process.Init.Minimal) !void {
     }
 
     var all_passed: bool = undefined;
+    const stdout_file = std.Io.File.stdout();
+    var stdout_buffer: [4096]u8 = undefined;
+    var reporter_io: std.Io.Threaded = .init(std.mem.Allocator.failing, .{ .environ = .empty });
+    defer reporter_io.deinit();
+    var stdout_writer = stdout_file.writer(reporter_io.io(), &stdout_buffer);
     const shard_options: ?lib.ShardOptions = if (cli_parser.options.shard_index) |index|
         .{ .index = index, .count = cli_parser.options.shard_count.? }
     else
@@ -206,6 +211,10 @@ pub fn main(init: std.process.Init.Minimal) !void {
             .verbose = cli_parser.options.verbose,
             .use_colors = !cli_parser.options.no_color,
             .shard = shard_options,
+            .reporter_type = cli_parser.options.reporter,
+            .junit_output = cli_parser.options.junit_output orelse "test-results.xml",
+            .timeout_ms = cli_parser.options.timeout,
+            .reporter_writer = &stdout_writer.interface,
             .coverage_options = cov_opts,
             .ui_server = if (ui_server) |*server| server else null,
         };
@@ -245,6 +254,10 @@ pub fn main(init: std.process.Init.Minimal) !void {
             .verbose = cli_parser.options.verbose,
             .use_colors = !cli_parser.options.no_color,
             .shard = shard_options,
+            .reporter_type = cli_parser.options.reporter,
+            .junit_output = cli_parser.options.junit_output orelse "test-results.xml",
+            .timeout_ms = cli_parser.options.timeout,
+            .reporter_writer = &stdout_writer.interface,
             .coverage_options = cov_opts,
             .ui_server = if (ui_server) |*server| server else null,
         };
@@ -256,7 +269,8 @@ pub fn main(init: std.process.Init.Minimal) !void {
         defer lib.cleanupRegistry();
 
         // Create test runner with CLI options
-        const runner_options = cli_parser.toRunnerOptions();
+        var runner_options = cli_parser.toRunnerOptions();
+        runner_options.reporter_writer = &stdout_writer.interface;
         var runner = lib.TestRunner.init(allocator, registry, runner_options);
         defer runner.deinit();
 

src/parallel.zig

@@ -1,6 +1,7 @@
 const std = @import("std");
 const suite = @import("suite.zig");
 const reporter = @import("reporter.zig");
+const pipeline = @import("pipeline.zig");
 const compat = @import("compat.zig");
 
 /// Options for bounded parallel test execution.
@@ -10,6 +11,7 @@ pub const ParallelOptions = struct {
     n_jobs: ?usize = null,
     enabled: bool = false,
     filter: ?[]const u8 = null,
+    timeout_ms: ?u64 = null,
 };
 
 pub const ParallelError = error{
@@ -31,6 +33,7 @@ const WorkerContext = struct {
     tests: []const *suite.TestCase,
     next_index: std.atomic.Value(usize) = std.atomic.Value(usize).init(0),
     error_mutex: compat.Mutex = .{},
+    policy: pipeline.ExecutionPolicy,
 
     fn worker(self: *WorkerContext) void {
         while (true) {
@@ -90,8 +93,15 @@ const WorkerContext = struct {
         test_case.error_message = std.fmt.allocPrint(self.allocator, format, args) catch "Out of memory";
     }
 
-    fn finishTiming(_: *WorkerContext, test_case: *suite.TestCase, start_time: i128) void {
+    fn finishTiming(self: *WorkerContext, test_case: *suite.TestCase, start_time: i128) void {
         test_case.execution_time_ns = @intCast(compat.nanoTimestamp() - start_time);
+        if (test_case.status == .passed and self.policy.timedOut(test_case.execution_time_ns)) {
+            self.fail(
+                test_case,
+                "Test exceeded timeout of {d}ms",
+                .{self.policy.timeout_ms.?},
+            );
+        }
     }
 };
 
@@ -167,6 +177,10 @@ const RunContext = struct {
             .allocator = self.allocator,
             .test_suite = test_suite,
             .tests = tests,
+            .policy = .{
+                .filter = self.options.filter,
+                .timeout_ms = self.options.timeout_ms,
+            },
         };
         const count = @min(self.worker_limit, tests.len);
         var threads: std.ArrayList(std.Thread) = .empty;