diff --git a/.github/workflows/benchmark_phpunit.yaml b/.github/workflows/benchmark_phpunit.yaml new file mode 100644 index 00000000000..c1bab0366ca --- /dev/null +++ b/.github/workflows/benchmark_phpunit.yaml @@ -0,0 +1,56 @@ +name: Benchmark PHPUnit + +# A/B wall-time benchmark: serial phpunit vs the Go parallel runner. +# Scheduled only (not on pull requests); runs every 2 hours on ubuntu + windows. +# Reminder: cron fires only from the default branch, so this starts running +# once the file is on `main`. +on: + schedule: + - cron: '0 */2 * * *' + workflow_dispatch: null + +env: + COMPOSER_ROOT_VERSION: "dev-main" + +jobs: + benchmark: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest] + php-versions: ['8.4', '8.5'] + + runs-on: ${{ matrix.os }} + timeout-minutes: 15 + + name: benchmark (${{ matrix.os }}) + steps: + - uses: actions/checkout@v5 + + - + uses: shivammathur/setup-php@v2 + with: + php-version: ${{ matrix.php-versions }} + coverage: none + ini-values: zend.assertions=1 + + - uses: "ramsey/composer-install@v4" + + - uses: actions/setup-go@v5 + with: + go-version: 'stable' + + - name: Serial phpunit + shell: bash + run: | + printf '| platform | php | mode | wall time |\n| --- | --- | --- | --- |\n' >> "$GITHUB_STEP_SUMMARY" + start=$SECONDS + vendor/bin/phpunit + echo "| ${{ matrix.os }} | ${{ matrix.php-versions }} | serial | $((SECONDS - start))s |" >> "$GITHUB_STEP_SUMMARY" + + - name: Go parallel runner + shell: bash + run: | + start=$SECONDS + go run ./utils-tests-runner/main.go + echo "| ${{ matrix.os }} | ${{ matrix.php-versions }} | go-runner | $((SECONDS - start))s |" >> "$GITHUB_STEP_SUMMARY" diff --git a/src/Testing/PHPUnit/AbstractRectorTestCase.php b/src/Testing/PHPUnit/AbstractRectorTestCase.php index 0380f667309..dc585fa10bf 100644 --- a/src/Testing/PHPUnit/AbstractRectorTestCase.php +++ b/src/Testing/PHPUnit/AbstractRectorTestCase.php @@ -30,6 +30,7 @@ use Rector\Testing\Fixture\FixtureSplitter; use Rector\Testing\PHPUnit\ValueObject\RectorTestResult; use Rector\Util\Reflection\PrivatesAccessor; +use Rector\ValueObject\PhpVersion; /** * @api used by public @@ -63,6 +64,10 @@ public static function tearDownAfterClass(): void SimpleParameterProvider::setParameter(Option::NEW_LINE_ON_FLUENT_CALL, false); SimpleParameterProvider::setParameter(Option::TREAT_CLASSES_AS_FINAL, false); + + // reset PHP version to the test default, so a version-bound test class + // does not leak its phpVersion() into the next class in the same process + SimpleParameterProvider::setParameter(Option::PHP_VERSION_FEATURES, PhpVersion::PHP_10); } protected function setUp(): void diff --git a/utils-tests-runner/.gitignore b/utils-tests-runner/.gitignore new file mode 100644 index 00000000000..e8d9d4d50e0 --- /dev/null +++ b/utils-tests-runner/.gitignore @@ -0,0 +1,2 @@ +/fast-phpunit +/fast-phpunit.exe diff --git a/utils-tests-runner/README.md b/utils-tests-runner/README.md new file mode 100644 index 00000000000..4b930c5113a --- /dev/null +++ b/utils-tests-runner/README.md @@ -0,0 +1,93 @@ +# fast-phpunit (proof of concept) + +Run the PHPUnit suite **~4x faster** by splitting test classes across parallel +workers that each boot PHP **once** and run many classes in a single process. + +## Speed — CI, per platform (GitHub runners, 4 vCPU, full suite) + +Serial `vendor/bin/phpunit` vs this runner, measured on the CI run step: + +| Platform | Serial | Go runner | Speedup | +| --- | --- | --- | --- | +| ubuntu-latest | 31s | **16s** | ~1.9x | +| windows-latest | **116s** | **67s** | **~1.7x** | + +Windows is the slow platform (~3.7x slower than Ubuntu serial); the runner cuts +~49s off it. Both run the full suite and pass. + +## Speed — local (24-core host, full suite) + +Full suite: **685 test classes / 5833 fixtures**. + +| Mode | Wall time | vs serial | +| --- | --- | --- | +| Serial `vendor/bin/phpunit` (1 process) | **38.3s** | 1.0x | +| `fast-phpunit -p 8` | ~11s | ~3.5x | +| `fast-phpunit -p 12` | **~9s** | **~4.2x** | +| `fast-phpunit -p 24` | ~9.8s | ~3.9x | + +Subset — `rules-tests/CodeQuality` (86 classes / 771 fixtures): + +| Mode | Wall time | +| --- | --- | +| Serial (1 process) | 5.71s | +| Process-per-class, `xargs -P8` | 8.55s (**slower than serial**) | +| `fast-phpunit -p 8` | **2.43s** | + +Sweet spot is ~12 workers; more does not help, because the heaviest chunk +bounds wall time and 20+ concurrent PHP processes start contending. + +## Why it is faster + +Bootstrap dominates a Rector test class, not the assertions: + +| Step | Time | +| --- | --- | +| Boot only (container build, 0 tests) | ~0.23s — fixed, per process | +| One class, 27 fixtures (warm) | 0.92s → ~26ms/fixture | + +Average class has ~7 fixtures, so **bootstrap is ~56% of an average class's run +time**. Any runner that spawns a fresh process per class pays that 0.23s boot +685 times — which is why process-per-class parallelism is slower than serial +(see subset table). + +This runner splits classes into N chunks balanced by fixture count, and each +worker runs its whole chunk in one warm process — so the container is built N +times, not 685 times. + +## Usage + +```bash +cd utils-tests-runner && go build -o fast-phpunit . +cd .. +utils-tests-runner/fast-phpunit -p 12 # whole suite +utils-tests-runner/fast-phpunit -p 8 rules-tests/CodeQuality # a subtree +``` + +Flags: `-p` workers (default = CPU count), `-bin` phpunit path. + +## Isolation — required to make it correct + +Two shared-state issues surface when many classes share a process; both are +handled so any chunking is safe. + +1. **Shared temp cache (cross-process).** Rector caches parsed files under + `sys_get_temp_dir()/rector_cached_files`; parallel processes racing that + directory throw `Failed to open directory` / `Directory not empty`. Each + worker gets its own `TMPDIR`. + +2. **Leaked `phpVersion()` (in-process) — a latent bug, fixed here.** + `phpVersion(...)` is stored in the static `SimpleParameterProvider` and was + never reset between classes, so a version-bound class leaks its version into + the next class in the same process — a version-less class then sees, e.g., + PHP 8.1 instead of the test default (`PhpVersion::PHP_10`) and produces wrong + output. The serial suite passes only because of its class ordering; any + reshuffle (this tool **or** paratest) can trigger it. Fixed in + `AbstractRectorTestCase::tearDownAfterClass()` by resetting + `PHP_VERSION_FEATURES` to the test default. + +## Status + +Proof of concept. Standalone Go helper that shells out to the existing +`vendor/bin/phpunit`, so it does not change how tests are written or how CI +runs. Pure Go, no `.php`, so it is invisible to ECS / PHPStan / Rector. diff --git a/utils-tests-runner/go.mod b/utils-tests-runner/go.mod new file mode 100644 index 00000000000..067b5c90707 --- /dev/null +++ b/utils-tests-runner/go.mod @@ -0,0 +1,3 @@ +module rector/fast-phpunit + +go 1.26.4 diff --git a/utils-tests-runner/main.go b/utils-tests-runner/main.go new file mode 100644 index 00000000000..ee03c9d4195 --- /dev/null +++ b/utils-tests-runner/main.go @@ -0,0 +1,181 @@ +// Command fast-phpunit runs the PHPUnit suite in parallel by splitting test +// classes into N balanced "warm" chunks: each worker boots PHP once and runs +// many test classes in a single process, so the container is built N times +// instead of once per class (as tools that spawn a process per chunk do). +// +// Balancing is by fixture count, since Rector rule tests iterate one assertion +// per .php.inc fixture, so fixture count approximates a class's runtime. +package main + +import ( + "flag" + "fmt" + "os" + "os/exec" + "path/filepath" + "regexp" + "runtime" + "sort" + "strings" + "sync" + "time" +) + +type testClass struct { + path string + weight int // fixture count, min 1 +} + +var fixtureCountRe = regexp.MustCompile(`\.php\.inc$`) + +func main() { + workers := flag.Int("p", runtime.NumCPU(), "number of parallel workers") + php := flag.String("php", "php", "php interpreter") + // the real PHP entry script (runs cross-platform via `php`); vendor/bin/phpunit + // is a shell/batch proxy on Windows and cannot be passed to php directly. + phpunit := flag.String("bin", "vendor/phpunit/phpunit/phpunit", "phpunit entry script") + flag.Parse() + + dirs := flag.Args() + if len(dirs) == 0 { + dirs = []string{"rules-tests", "tests"} + } + + classes := discover(dirs) + if len(classes) == 0 { + fmt.Fprintln(os.Stderr, "no test classes found") + os.Exit(1) + } + + chunks := balance(classes, *workers) + + start := time.Now() + failed := run(chunks, *php, *phpunit, *workers) + elapsed := time.Since(start) + + totalFixtures := 0 + for _, c := range classes { + totalFixtures += c.weight + } + fmt.Printf("\n%d classes, %d fixtures, %d chunks, %d workers\n", + len(classes), totalFixtures, len(chunks), *workers) + fmt.Printf("wall time: %.2fs\n", elapsed.Seconds()) + + if failed > 0 { + fmt.Printf("FAILED chunks: %d\n", failed) + os.Exit(1) + } + fmt.Println("OK") +} + +// discover finds *Test.php files and weights each by sibling Fixture/ file count. +func discover(dirs []string) []testClass { + var classes []testClass + for _, dir := range dirs { + _ = filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error { + if err != nil || d.IsDir() || !strings.HasSuffix(path, "Test.php") { + return nil + } + classes = append(classes, testClass{path: path, weight: fixtureWeight(path)}) + return nil + }) + } + return classes +} + +func fixtureWeight(testPath string) int { + fixtureDir := filepath.Join(filepath.Dir(testPath), "Fixture") + entries, err := os.ReadDir(fixtureDir) + if err != nil { + return 1 + } + count := 0 + for _, e := range entries { + if !e.IsDir() && fixtureCountRe.MatchString(e.Name()) { + count++ + } + } + if count < 1 { + return 1 + } + return count +} + +// balance greedily packs classes (heaviest first) into n bins, always adding to +// the lightest bin. Minimizes the heaviest chunk, so wall time is bounded by the +// slowest worker rather than by unlucky static splits. +func balance(classes []testClass, n int) [][]testClass { + sort.Slice(classes, func(i, j int) bool { + return classes[i].weight > classes[j].weight + }) + bins := make([][]testClass, n) + loads := make([]int, n) + for _, c := range classes { + min := 0 + for i := 1; i < n; i++ { + if loads[i] < loads[min] { + min = i + } + } + bins[min] = append(bins[min], c) + loads[min] += c.weight + } + var out [][]testClass + for _, b := range bins { + if len(b) > 0 { + out = append(out, b) + } + } + return out +} + +func run(chunks [][]testClass, php, phpunit string, workers int) int { + sem := make(chan struct{}, workers) + var wg sync.WaitGroup + var mu sync.Mutex + failed := 0 + + for idx, chunk := range chunks { + wg.Add(1) + go func(idx int, chunk []testClass) { + defer wg.Done() + sem <- struct{}{} + defer func() { <-sem }() + + // Each worker gets its own temp dir so Rector's file cache + // (sys_get_temp_dir()/rector_cached_files) and the fixture temp + // dumper never race across processes. sys_get_temp_dir() reads + // TMPDIR on Linux/macOS and TMP/TEMP on Windows, so set all three. + tmp := filepath.Join(os.TempDir(), fmt.Sprintf("fast-phpunit-%d", idx)) + _ = os.MkdirAll(tmp, 0o755) + defer os.RemoveAll(tmp) + + // invoke via `php ` so it works uniformly on Windows, + // where vendor/bin/phpunit is not directly executable. + args := make([]string, 0, len(chunk)+1) + args = append(args, phpunit) + for _, c := range chunk { + args = append(args, c.path) + } + cmd := exec.Command(php, args...) + cmd.Env = append(os.Environ(), "TMPDIR="+tmp, "TMP="+tmp, "TEMP="+tmp) + out, err := cmd.CombinedOutput() + if err != nil { + mu.Lock() + failed++ + fmt.Printf("chunk FAILED (%d classes): %v\n%s\n", len(chunk), err, tail(string(out), 15)) + mu.Unlock() + } + }(idx, chunk) + } + wg.Wait() + return failed +} + +func tail(s string, lines int) string { + parts := strings.Split(strings.TrimRight(s, "\n"), "\n") + if len(parts) > lines { + parts = parts[len(parts)-lines:] + } + return strings.Join(parts, "\n") +}