[PoC] Parallel PHPUnit runner in Go (~4x faster) + fix phpVersion leak between test classes

.github/workflows/benchmark_phpunit.yaml

@@ -0,0 +1,56 @@
+name: Benchmark PHPUnit
+
+# A/B wall-time benchmark: serial phpunit vs the Go parallel runner.
+# Scheduled only (not on pull requests); runs every 2 hours on ubuntu + windows.
+# Reminder: cron fires only from the default branch, so this starts running
+# once the file is on `main`.
+on:
+    schedule:
+        - cron: '0 */2 * * *'
+    workflow_dispatch: null
+
+env:
+    COMPOSER_ROOT_VERSION: "dev-main"
+
+jobs:
+    benchmark:
+        strategy:
+            fail-fast: false
+            matrix:
+                os: [ubuntu-latest, windows-latest]
+                php-versions: ['8.4', '8.5']
+
+        runs-on: ${{ matrix.os }}
+        timeout-minutes: 15
+
+        name: benchmark (${{ matrix.os }})
+        steps:
+            -   uses: actions/checkout@v5
+
+            -
+                uses: shivammathur/setup-php@v2
+                with:
+                    php-version: ${{ matrix.php-versions }}
+                    coverage: none
+                    ini-values: zend.assertions=1
+
+            -   uses: "ramsey/composer-install@v4"
+
+            -   uses: actions/setup-go@v5
+                with:
+                    go-version: 'stable'
+
+            -   name: Serial phpunit
+                shell: bash
+                run: |
+                    printf '| platform | php | mode | wall time |\n| --- | --- | --- | --- |\n' >> "$GITHUB_STEP_SUMMARY"
+                    start=$SECONDS
+                    vendor/bin/phpunit
+                    echo "| ${{ matrix.os }} | ${{ matrix.php-versions }} | serial | $((SECONDS - start))s |" >> "$GITHUB_STEP_SUMMARY"
+
+            -   name: Go parallel runner
+                shell: bash
+                run: |
+                    start=$SECONDS
+                    go run ./utils-tests-runner/main.go
+                    echo "| ${{ matrix.os }} | ${{ matrix.php-versions }} | go-runner | $((SECONDS - start))s |" >> "$GITHUB_STEP_SUMMARY"

src/Testing/PHPUnit/AbstractRectorTestCase.php

@@ -30,6 +30,7 @@
 use Rector\Testing\Fixture\FixtureSplitter;
 use Rector\Testing\PHPUnit\ValueObject\RectorTestResult;
 use Rector\Util\Reflection\PrivatesAccessor;
+use Rector\ValueObject\PhpVersion;
 
 /**
  * @api used by public
@@ -63,6 +64,10 @@ public static function tearDownAfterClass(): void
         SimpleParameterProvider::setParameter(Option::NEW_LINE_ON_FLUENT_CALL, false);
 
         SimpleParameterProvider::setParameter(Option::TREAT_CLASSES_AS_FINAL, false);
+
+        // reset PHP version to the test default, so a version-bound test class
+        // does not leak its phpVersion() into the next class in the same process
+        SimpleParameterProvider::setParameter(Option::PHP_VERSION_FEATURES, PhpVersion::PHP_10);
     }
 
     protected function setUp(): void

utils-tests-runner/.gitignore

@@ -0,0 +1,2 @@
+/fast-phpunit
+/fast-phpunit.exe

utils-tests-runner/README.md

@@ -0,0 +1,93 @@
+# fast-phpunit (proof of concept)
+
+Run the PHPUnit suite **~4x faster** by splitting test classes across parallel
+workers that each boot PHP **once** and run many classes in a single process.
+
+## Speed — CI, per platform (GitHub runners, 4 vCPU, full suite)
+
+Serial `vendor/bin/phpunit` vs this runner, measured on the CI run step:
+
+| Platform | Serial | Go runner | Speedup |
+| --- | --- | --- | --- |
+| ubuntu-latest | 31s | **16s** | ~1.9x |
+| windows-latest | **116s** | **67s** | **~1.7x** |
+
+Windows is the slow platform (~3.7x slower than Ubuntu serial); the runner cuts
+~49s off it. Both run the full suite and pass.
+
+## Speed — local (24-core host, full suite)
+
+Full suite: **685 test classes / 5833 fixtures**.
+
+| Mode | Wall time | vs serial |
+| --- | --- | --- |
+| Serial `vendor/bin/phpunit` (1 process) | **38.3s** | 1.0x |
+| `fast-phpunit -p 8` | ~11s | ~3.5x |
+| `fast-phpunit -p 12` | **~9s** | **~4.2x** |
+| `fast-phpunit -p 24` | ~9.8s | ~3.9x |
+
+Subset — `rules-tests/CodeQuality` (86 classes / 771 fixtures):
+
+| Mode | Wall time |
+| --- | --- |
+| Serial (1 process) | 5.71s |
+| Process-per-class, `xargs -P8` | 8.55s (**slower than serial**) |
+| `fast-phpunit -p 8` | **2.43s** |
+
+Sweet spot is ~12 workers; more does not help, because the heaviest chunk
+bounds wall time and 20+ concurrent PHP processes start contending.
+
+## Why it is faster
+
+Bootstrap dominates a Rector test class, not the assertions:
+
+| Step | Time |
+| --- | --- |
+| Boot only (container build, 0 tests) | ~0.23s — fixed, per process |
+| One class, 27 fixtures (warm) | 0.92s → ~26ms/fixture |
+
+Average class has ~7 fixtures, so **bootstrap is ~56% of an average class's run
+time**. Any runner that spawns a fresh process per class pays that 0.23s boot
+685 times — which is why process-per-class parallelism is slower than serial
+(see subset table).
+
+This runner splits classes into N chunks balanced by fixture count, and each
+worker runs its whole chunk in one warm process — so the container is built N
+times, not 685 times.
+
+## Usage
+
+```bash
+cd utils-tests-runner && go build -o fast-phpunit .
+cd ..
+utils-tests-runner/fast-phpunit -p 12                          # whole suite
+utils-tests-runner/fast-phpunit -p 8 rules-tests/CodeQuality   # a subtree
+```
+
+Flags: `-p` workers (default = CPU count), `-bin` phpunit path.
+
+## Isolation — required to make it correct
+
+Two shared-state issues surface when many classes share a process; both are
+handled so any chunking is safe.
+
+1. **Shared temp cache (cross-process).** Rector caches parsed files under
+   `sys_get_temp_dir()/rector_cached_files`; parallel processes racing that
+   directory throw `Failed to open directory` / `Directory not empty`. Each
+   worker gets its own `TMPDIR`.
+
+2. **Leaked `phpVersion()` (in-process) — a latent bug, fixed here.**
+   `phpVersion(...)` is stored in the static `SimpleParameterProvider` and was
+   never reset between classes, so a version-bound class leaks its version into
+   the next class in the same process — a version-less class then sees, e.g.,
+   PHP 8.1 instead of the test default (`PhpVersion::PHP_10`) and produces wrong
+   output. The serial suite passes only because of its class ordering; any
+   reshuffle (this tool **or** paratest) can trigger it. Fixed in
+   `AbstractRectorTestCase::tearDownAfterClass()` by resetting
+   `PHP_VERSION_FEATURES` to the test default.
+
+## Status
+
+Proof of concept. Standalone Go helper that shells out to the existing
+`vendor/bin/phpunit`, so it does not change how tests are written or how CI
+runs. Pure Go, no `.php`, so it is invisible to ECS / PHPStan / Rector.

utils-tests-runner/go.mod

@@ -0,0 +1,3 @@
+module rector/fast-phpunit
+
+go 1.26.4

utils-tests-runner/main.go

@@ -0,0 +1,181 @@
+// Command fast-phpunit runs the PHPUnit suite in parallel by splitting test
+// classes into N balanced "warm" chunks: each worker boots PHP once and runs
+// many test classes in a single process, so the container is built N times
+// instead of once per class (as tools that spawn a process per chunk do).
+//
+// Balancing is by fixture count, since Rector rule tests iterate one assertion
+// per .php.inc fixture, so fixture count approximates a class's runtime.
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+	"runtime"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+)
+
+type testClass struct {
+	path   string
+	weight int // fixture count, min 1
+}
+
+var fixtureCountRe = regexp.MustCompile(`\.php\.inc$`)
+
+func main() {
+	workers := flag.Int("p", runtime.NumCPU(), "number of parallel workers")
+	php := flag.String("php", "php", "php interpreter")
+	// the real PHP entry script (runs cross-platform via `php`); vendor/bin/phpunit
+	// is a shell/batch proxy on Windows and cannot be passed to php directly.
+	phpunit := flag.String("bin", "vendor/phpunit/phpunit/phpunit", "phpunit entry script")
+	flag.Parse()
+
+	dirs := flag.Args()
+	if len(dirs) == 0 {
+		dirs = []string{"rules-tests", "tests"}
+	}
+
+	classes := discover(dirs)
+	if len(classes) == 0 {
+		fmt.Fprintln(os.Stderr, "no test classes found")
+		os.Exit(1)
+	}
+
+	chunks := balance(classes, *workers)
+
+	start := time.Now()
+	failed := run(chunks, *php, *phpunit, *workers)
+	elapsed := time.Since(start)
+
+	totalFixtures := 0
+	for _, c := range classes {
+		totalFixtures += c.weight
+	}
+	fmt.Printf("\n%d classes, %d fixtures, %d chunks, %d workers\n",
+		len(classes), totalFixtures, len(chunks), *workers)
+	fmt.Printf("wall time: %.2fs\n", elapsed.Seconds())
+
+	if failed > 0 {
+		fmt.Printf("FAILED chunks: %d\n", failed)
+		os.Exit(1)
+	}
+	fmt.Println("OK")
+}
+
+// discover finds *Test.php files and weights each by sibling Fixture/ file count.
+func discover(dirs []string) []testClass {
+	var classes []testClass
+	for _, dir := range dirs {
+		_ = filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error {
+			if err != nil || d.IsDir() || !strings.HasSuffix(path, "Test.php") {
+				return nil
+			}
+			classes = append(classes, testClass{path: path, weight: fixtureWeight(path)})
+			return nil
+		})
+	}
+	return classes
+}
+
+func fixtureWeight(testPath string) int {
+	fixtureDir := filepath.Join(filepath.Dir(testPath), "Fixture")
+	entries, err := os.ReadDir(fixtureDir)
+	if err != nil {
+		return 1
+	}
+	count := 0
+	for _, e := range entries {
+		if !e.IsDir() && fixtureCountRe.MatchString(e.Name()) {
+			count++
+		}
+	}
+	if count < 1 {
+		return 1
+	}
+	return count
+}
+
+// balance greedily packs classes (heaviest first) into n bins, always adding to
+// the lightest bin. Minimizes the heaviest chunk, so wall time is bounded by the
+// slowest worker rather than by unlucky static splits.
+func balance(classes []testClass, n int) [][]testClass {
+	sort.Slice(classes, func(i, j int) bool {
+		return classes[i].weight > classes[j].weight
+	})
+	bins := make([][]testClass, n)
+	loads := make([]int, n)
+	for _, c := range classes {
+		min := 0
+		for i := 1; i < n; i++ {
+			if loads[i] < loads[min] {
+				min = i
+			}
+		}
+		bins[min] = append(bins[min], c)
+		loads[min] += c.weight
+	}
+	var out [][]testClass
+	for _, b := range bins {
+		if len(b) > 0 {
+			out = append(out, b)
+		}
+	}
+	return out
+}
+
+func run(chunks [][]testClass, php, phpunit string, workers int) int {
+	sem := make(chan struct{}, workers)
+	var wg sync.WaitGroup
+	var mu sync.Mutex
+	failed := 0
+
+	for idx, chunk := range chunks {
+		wg.Add(1)
+		go func(idx int, chunk []testClass) {
+			defer wg.Done()
+			sem <- struct{}{}
+			defer func() { <-sem }()
+
+			// Each worker gets its own temp dir so Rector's file cache
+			// (sys_get_temp_dir()/rector_cached_files) and the fixture temp
+			// dumper never race across processes. sys_get_temp_dir() reads
+			// TMPDIR on Linux/macOS and TMP/TEMP on Windows, so set all three.
+			tmp := filepath.Join(os.TempDir(), fmt.Sprintf("fast-phpunit-%d", idx))
+			_ = os.MkdirAll(tmp, 0o755)
+			defer os.RemoveAll(tmp)
+
+			// invoke via `php <phpunit>` so it works uniformly on Windows,
+			// where vendor/bin/phpunit is not directly executable.
+			args := make([]string, 0, len(chunk)+1)
+			args = append(args, phpunit)
+			for _, c := range chunk {
+				args = append(args, c.path)
+			}
+			cmd := exec.Command(php, args...)
+			cmd.Env = append(os.Environ(), "TMPDIR="+tmp, "TMP="+tmp, "TEMP="+tmp)
+			out, err := cmd.CombinedOutput()
+			if err != nil {
+				mu.Lock()
+				failed++
+				fmt.Printf("chunk FAILED (%d classes): %v\n%s\n", len(chunk), err, tail(string(out), 15))
+				mu.Unlock()
+			}
+		}(idx, chunk)
+	}
+	wg.Wait()
+	return failed
+}
+
+func tail(s string, lines int) string {
+	parts := strings.Split(strings.TrimRight(s, "\n"), "\n")
+	if len(parts) > lines {
+		parts = parts[len(parts)-lines:]
+	}
+	return strings.Join(parts, "\n")
+}