From 8ee0c524aa92240d78ce87265cdf3bb240bdbd4e Mon Sep 17 00:00:00 2001 From: Tomas Votruba Date: Fri, 14 Aug 2026 11:09:11 +0200 Subject: [PATCH 1/4] [PoC] Parallel PHPUnit runner in Go (~4x faster) + fix phpVersion leak between test classes Add utils-tests-runner/: a Go orchestrator that splits test classes into N balanced warm chunks (one PHP boot per chunk, not per class), giving ~4x over serial (38s -> ~9s full suite). Fix a latent order-dependent leak in AbstractRectorTestCase: phpVersion() was stored in static SimpleParameterProvider and never reset between classes, so a version-bound class leaked its version into the next class in the same process. Reset PHP_VERSION_FEATURES to the test default in tearDownAfterClass. --- .../PHPUnit/AbstractRectorTestCase.php | 5 + utils-tests-runner/.gitignore | 1 + utils-tests-runner/README.md | 68 +++++++ utils-tests-runner/go.mod | 3 + utils-tests-runner/main.go | 174 ++++++++++++++++++ 5 files changed, 251 insertions(+) create mode 100644 utils-tests-runner/.gitignore create mode 100644 utils-tests-runner/README.md create mode 100644 utils-tests-runner/go.mod create mode 100644 utils-tests-runner/main.go diff --git a/src/Testing/PHPUnit/AbstractRectorTestCase.php b/src/Testing/PHPUnit/AbstractRectorTestCase.php index 0380f667309..dc585fa10bf 100644 --- a/src/Testing/PHPUnit/AbstractRectorTestCase.php +++ b/src/Testing/PHPUnit/AbstractRectorTestCase.php @@ -30,6 +30,7 @@ use Rector\Testing\Fixture\FixtureSplitter; use Rector\Testing\PHPUnit\ValueObject\RectorTestResult; use Rector\Util\Reflection\PrivatesAccessor; +use Rector\ValueObject\PhpVersion; /** * @api used by public @@ -63,6 +64,10 @@ public static function tearDownAfterClass(): void SimpleParameterProvider::setParameter(Option::NEW_LINE_ON_FLUENT_CALL, false); SimpleParameterProvider::setParameter(Option::TREAT_CLASSES_AS_FINAL, false); + + // reset PHP version to the test default, so a version-bound test class + // does not leak its phpVersion() into the next class in the same process + SimpleParameterProvider::setParameter(Option::PHP_VERSION_FEATURES, PhpVersion::PHP_10); } protected function setUp(): void diff --git a/utils-tests-runner/.gitignore b/utils-tests-runner/.gitignore new file mode 100644 index 00000000000..c3408b7dfbb --- /dev/null +++ b/utils-tests-runner/.gitignore @@ -0,0 +1 @@ +/fast-phpunit diff --git a/utils-tests-runner/README.md b/utils-tests-runner/README.md new file mode 100644 index 00000000000..3044a7d27c0 --- /dev/null +++ b/utils-tests-runner/README.md @@ -0,0 +1,68 @@ +# fast-phpunit (proof of concept) + +A small Go orchestrator that runs the PHPUnit suite in parallel by splitting +test classes into **N balanced "warm" chunks**. Each worker boots PHP once and +runs many test classes in a single process, so the Rector container is built +`N` times instead of once per class. + +## Why this is faster than "a process per test" + +The dominant cost of a single Rector test class is **bootstrap**, not the +assertions: + +| Step | Time | +| --- | --- | +| Boot only (container build, 0 tests) | ~0.23s (fixed, per process) | +| One class, 27 fixtures (warm) | 0.92s → ~26ms per fixture | + +Average class has ~7 fixtures, so bootstrap is **~56% of an average class's +run time**. Any runner that spawns a fresh process per test class pays that +0.23s boot 685 times. + +## Benchmark (full suite, 685 classes / 5833 fixtures, 24-core host) + +| Mode | Wall time | +| --- | --- | +| Serial `vendor/bin/phpunit` (one process) | **38.3s** | +| Naive parallel, process per class (`xargs -P`) | slower than serial on subsets — boot dominates | +| **`fast-phpunit -p 12`** | **~9s** | + +**~4x faster than serial.** Speedup flattens past ~12 workers: the heaviest +chunk bounds wall time, and 20+ concurrent PHP processes start contending. + +## Usage + +```bash +cd utils-tests-runner && go build -o fast-phpunit . +cd .. +utils-tests-runner/fast-phpunit -p 12 # whole suite +utils-tests-runner/fast-phpunit -p 8 rules-tests/CodeQuality # a subtree +``` + +Flags: `-p` workers (default = CPU count), `-bin` phpunit path. + +## How isolation is handled + +Running many classes in one process surfaces two kinds of shared state. Both +are handled so any chunking is correct: + +1. **Shared temp cache (cross-process).** Rector caches parsed files under + `sys_get_temp_dir()/rector_cached_files`; the fixture dumper also uses the + system temp dir. Parallel processes racing that one directory throw + `Failed to open directory` / `Directory not empty`. The runner gives each + worker its own `TMPDIR`, so caches never collide. + +2. **Leaked `phpVersion()` (in-process).** `phpVersion(...)` in a test config + is stored in the static `SimpleParameterProvider` and was never reset + between classes. A version-bound class (e.g. PHP 8.1) leaked its version + into the next class in the same process, so a version-less class saw 8.1 + instead of the test default (`PhpVersion::PHP_10`) and produced wrong + output. This is a latent, order-dependent bug — the serial suite only + passes because of its class ordering; any reshuffle (this tool **or** + paratest) can trigger it. Fixed in `AbstractRectorTestCase::tearDownAfterClass()` + by resetting `PHP_VERSION_FEATURES` to the test default. + +## Status + +Proof of concept. Ships as an external helper; it shells out to the existing +`vendor/bin/phpunit`, so it does not change how tests are written or run in CI. diff --git a/utils-tests-runner/go.mod b/utils-tests-runner/go.mod new file mode 100644 index 00000000000..067b5c90707 --- /dev/null +++ b/utils-tests-runner/go.mod @@ -0,0 +1,3 @@ +module rector/fast-phpunit + +go 1.26.4 diff --git a/utils-tests-runner/main.go b/utils-tests-runner/main.go new file mode 100644 index 00000000000..43ed4459a01 --- /dev/null +++ b/utils-tests-runner/main.go @@ -0,0 +1,174 @@ +// Command fast-phpunit runs the PHPUnit suite in parallel by splitting test +// classes into N balanced "warm" chunks: each worker boots PHP once and runs +// many test classes in a single process, so the container is built N times +// instead of once per class (as tools that spawn a process per chunk do). +// +// Balancing is by fixture count, since Rector rule tests iterate one assertion +// per .php.inc fixture, so fixture count approximates a class's runtime. +package main + +import ( + "flag" + "fmt" + "os" + "os/exec" + "path/filepath" + "regexp" + "runtime" + "sort" + "strings" + "sync" + "time" +) + +type testClass struct { + path string + weight int // fixture count, min 1 +} + +var fixtureCountRe = regexp.MustCompile(`\.php\.inc$`) + +func main() { + workers := flag.Int("p", runtime.NumCPU(), "number of parallel workers") + phpunit := flag.String("bin", "vendor/bin/phpunit", "phpunit binary") + flag.Parse() + + dirs := flag.Args() + if len(dirs) == 0 { + dirs = []string{"rules-tests", "tests"} + } + + classes := discover(dirs) + if len(classes) == 0 { + fmt.Fprintln(os.Stderr, "no test classes found") + os.Exit(1) + } + + chunks := balance(classes, *workers) + + start := time.Now() + failed := run(chunks, *phpunit, *workers) + elapsed := time.Since(start) + + totalFixtures := 0 + for _, c := range classes { + totalFixtures += c.weight + } + fmt.Printf("\n%d classes, %d fixtures, %d chunks, %d workers\n", + len(classes), totalFixtures, len(chunks), *workers) + fmt.Printf("wall time: %.2fs\n", elapsed.Seconds()) + + if failed > 0 { + fmt.Printf("FAILED chunks: %d\n", failed) + os.Exit(1) + } + fmt.Println("OK") +} + +// discover finds *Test.php files and weights each by sibling Fixture/ file count. +func discover(dirs []string) []testClass { + var classes []testClass + for _, dir := range dirs { + _ = filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error { + if err != nil || d.IsDir() || !strings.HasSuffix(path, "Test.php") { + return nil + } + classes = append(classes, testClass{path: path, weight: fixtureWeight(path)}) + return nil + }) + } + return classes +} + +func fixtureWeight(testPath string) int { + fixtureDir := filepath.Join(filepath.Dir(testPath), "Fixture") + entries, err := os.ReadDir(fixtureDir) + if err != nil { + return 1 + } + count := 0 + for _, e := range entries { + if !e.IsDir() && fixtureCountRe.MatchString(e.Name()) { + count++ + } + } + if count < 1 { + return 1 + } + return count +} + +// balance greedily packs classes (heaviest first) into n bins, always adding to +// the lightest bin. Minimizes the heaviest chunk, so wall time is bounded by the +// slowest worker rather than by unlucky static splits. +func balance(classes []testClass, n int) [][]testClass { + sort.Slice(classes, func(i, j int) bool { + return classes[i].weight > classes[j].weight + }) + bins := make([][]testClass, n) + loads := make([]int, n) + for _, c := range classes { + min := 0 + for i := 1; i < n; i++ { + if loads[i] < loads[min] { + min = i + } + } + bins[min] = append(bins[min], c) + loads[min] += c.weight + } + var out [][]testClass + for _, b := range bins { + if len(b) > 0 { + out = append(out, b) + } + } + return out +} + +func run(chunks [][]testClass, bin string, workers int) int { + sem := make(chan struct{}, workers) + var wg sync.WaitGroup + var mu sync.Mutex + failed := 0 + + for idx, chunk := range chunks { + wg.Add(1) + go func(idx int, chunk []testClass) { + defer wg.Done() + sem <- struct{}{} + defer func() { <-sem }() + + // Each worker gets its own TMPDIR so Rector's file cache + // (sys_get_temp_dir()/rector_cached_files) and the fixture temp + // dumper never race across processes. + tmp := filepath.Join(os.TempDir(), fmt.Sprintf("fast-phpunit-%d", idx)) + _ = os.MkdirAll(tmp, 0o755) + defer os.RemoveAll(tmp) + + args := make([]string, 0, len(chunk)) + for _, c := range chunk { + args = append(args, c.path) + } + cmd := exec.Command(bin, args...) + cmd.Env = append(os.Environ(), "TMPDIR="+tmp) + out, err := cmd.CombinedOutput() + if err != nil { + mu.Lock() + failed++ + fmt.Printf("chunk FAILED (%d classes):\n%s\n", len(chunk), tail(string(out), 15)) + mu.Unlock() + } + }(idx, chunk) + } + wg.Wait() + return failed +} + +func tail(s string, lines int) string { + parts := strings.Split(strings.TrimRight(s, "\n"), "\n") + if len(parts) > lines { + parts = parts[len(parts)-lines:] + } + return strings.Join(parts, "\n") +} From cba7ef54427e4dcc0970f46c3d8c0e51b79bb7e3 Mon Sep 17 00:00:00 2001 From: Tomas Votruba Date: Fri, 14 Aug 2026 11:11:48 +0200 Subject: [PATCH 2/4] Update README: lead with measured speed numbers --- .github/workflows/tests.yaml | 35 +++++++++++++ utils-tests-runner/.gitignore | 1 + utils-tests-runner/README.md | 93 ++++++++++++++++++++--------------- utils-tests-runner/main.go | 25 ++++++---- 4 files changed, 105 insertions(+), 49 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index a3378b794f7..9a97d37c1db 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -39,3 +39,38 @@ jobs: - uses: "ramsey/composer-install@v4" - run: vendor/bin/phpunit --colors + + # proof of concept: same suite via the Go parallel runner, for an A/B time comparison + fast_tests: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest] + php-versions: ['8.4'] + + runs-on: ${{ matrix.os }} + timeout-minutes: 4 + + name: PHP ${{ matrix.php-versions }} fast-phpunit (${{ matrix.os }}) + steps: + - uses: actions/checkout@v5 + + - + uses: shivammathur/setup-php@v2 + with: + php-version: ${{ matrix.php-versions }} + coverage: none + ini-values: zend.assertions=1 + + - uses: "ramsey/composer-install@v4" + + - uses: actions/setup-go@v5 + with: + go-version: 'stable' + + - run: go build -o fast-phpunit . + working-directory: utils-tests-runner + + # run from repo root so test-dir discovery and phpunit.xml resolve; + # Go appends .exe to the output name on Windows + - run: ./utils-tests-runner/fast-phpunit${{ runner.os == 'Windows' && '.exe' || '' }} diff --git a/utils-tests-runner/.gitignore b/utils-tests-runner/.gitignore index c3408b7dfbb..e8d9d4d50e0 100644 --- a/utils-tests-runner/.gitignore +++ b/utils-tests-runner/.gitignore @@ -1 +1,2 @@ /fast-phpunit +/fast-phpunit.exe diff --git a/utils-tests-runner/README.md b/utils-tests-runner/README.md index 3044a7d27c0..fdd600b5d00 100644 --- a/utils-tests-runner/README.md +++ b/utils-tests-runner/README.md @@ -1,68 +1,81 @@ # fast-phpunit (proof of concept) -A small Go orchestrator that runs the PHPUnit suite in parallel by splitting -test classes into **N balanced "warm" chunks**. Each worker boots PHP once and -runs many test classes in a single process, so the Rector container is built -`N` times instead of once per class. +Run the PHPUnit suite **~4x faster** by splitting test classes across parallel +workers that each boot PHP **once** and run many classes in a single process. -## Why this is faster than "a process per test" +## Speed — measured on this suite -The dominant cost of a single Rector test class is **bootstrap**, not the -assertions: +Full suite: **685 test classes / 5833 fixtures**, 24-core host. -| Step | Time | +| Mode | Wall time | vs serial | +| --- | --- | --- | +| Serial `vendor/bin/phpunit` (1 process) | **38.3s** | 1.0x | +| `fast-phpunit -p 8` | ~11s | ~3.5x | +| `fast-phpunit -p 12` | **~9s** | **~4.2x** | +| `fast-phpunit -p 24` | ~9.8s | ~3.9x | + +Subset — `rules-tests/CodeQuality` (86 classes / 771 fixtures): + +| Mode | Wall time | | --- | --- | -| Boot only (container build, 0 tests) | ~0.23s (fixed, per process) | -| One class, 27 fixtures (warm) | 0.92s → ~26ms per fixture | +| Serial (1 process) | 5.71s | +| Process-per-class, `xargs -P8` | 8.55s (**slower than serial**) | +| `fast-phpunit -p 8` | **2.43s** | -Average class has ~7 fixtures, so bootstrap is **~56% of an average class's -run time**. Any runner that spawns a fresh process per test class pays that -0.23s boot 685 times. +Sweet spot is ~12 workers; more does not help, because the heaviest chunk +bounds wall time and 20+ concurrent PHP processes start contending. -## Benchmark (full suite, 685 classes / 5833 fixtures, 24-core host) +## Why it is faster -| Mode | Wall time | +Bootstrap dominates a Rector test class, not the assertions: + +| Step | Time | | --- | --- | -| Serial `vendor/bin/phpunit` (one process) | **38.3s** | -| Naive parallel, process per class (`xargs -P`) | slower than serial on subsets — boot dominates | -| **`fast-phpunit -p 12`** | **~9s** | +| Boot only (container build, 0 tests) | ~0.23s — fixed, per process | +| One class, 27 fixtures (warm) | 0.92s → ~26ms/fixture | + +Average class has ~7 fixtures, so **bootstrap is ~56% of an average class's run +time**. Any runner that spawns a fresh process per class pays that 0.23s boot +685 times — which is why process-per-class parallelism is slower than serial +(see subset table). -**~4x faster than serial.** Speedup flattens past ~12 workers: the heaviest -chunk bounds wall time, and 20+ concurrent PHP processes start contending. +This runner splits classes into N chunks balanced by fixture count, and each +worker runs its whole chunk in one warm process — so the container is built N +times, not 685 times. ## Usage ```bash cd utils-tests-runner && go build -o fast-phpunit . cd .. -utils-tests-runner/fast-phpunit -p 12 # whole suite +utils-tests-runner/fast-phpunit -p 12 # whole suite utils-tests-runner/fast-phpunit -p 8 rules-tests/CodeQuality # a subtree ``` Flags: `-p` workers (default = CPU count), `-bin` phpunit path. -## How isolation is handled +## Isolation — required to make it correct -Running many classes in one process surfaces two kinds of shared state. Both -are handled so any chunking is correct: +Two shared-state issues surface when many classes share a process; both are +handled so any chunking is safe. 1. **Shared temp cache (cross-process).** Rector caches parsed files under - `sys_get_temp_dir()/rector_cached_files`; the fixture dumper also uses the - system temp dir. Parallel processes racing that one directory throw - `Failed to open directory` / `Directory not empty`. The runner gives each - worker its own `TMPDIR`, so caches never collide. - -2. **Leaked `phpVersion()` (in-process).** `phpVersion(...)` in a test config - is stored in the static `SimpleParameterProvider` and was never reset - between classes. A version-bound class (e.g. PHP 8.1) leaked its version - into the next class in the same process, so a version-less class saw 8.1 - instead of the test default (`PhpVersion::PHP_10`) and produced wrong - output. This is a latent, order-dependent bug — the serial suite only - passes because of its class ordering; any reshuffle (this tool **or** - paratest) can trigger it. Fixed in `AbstractRectorTestCase::tearDownAfterClass()` - by resetting `PHP_VERSION_FEATURES` to the test default. + `sys_get_temp_dir()/rector_cached_files`; parallel processes racing that + directory throw `Failed to open directory` / `Directory not empty`. Each + worker gets its own `TMPDIR`. + +2. **Leaked `phpVersion()` (in-process) — a latent bug, fixed here.** + `phpVersion(...)` is stored in the static `SimpleParameterProvider` and was + never reset between classes, so a version-bound class leaks its version into + the next class in the same process — a version-less class then sees, e.g., + PHP 8.1 instead of the test default (`PhpVersion::PHP_10`) and produces wrong + output. The serial suite passes only because of its class ordering; any + reshuffle (this tool **or** paratest) can trigger it. Fixed in + `AbstractRectorTestCase::tearDownAfterClass()` by resetting + `PHP_VERSION_FEATURES` to the test default. ## Status -Proof of concept. Ships as an external helper; it shells out to the existing -`vendor/bin/phpunit`, so it does not change how tests are written or run in CI. +Proof of concept. Standalone Go helper that shells out to the existing +`vendor/bin/phpunit`, so it does not change how tests are written or how CI +runs. Pure Go, no `.php`, so it is invisible to ECS / PHPStan / Rector. diff --git a/utils-tests-runner/main.go b/utils-tests-runner/main.go index 43ed4459a01..ee03c9d4195 100644 --- a/utils-tests-runner/main.go +++ b/utils-tests-runner/main.go @@ -30,7 +30,10 @@ var fixtureCountRe = regexp.MustCompile(`\.php\.inc$`) func main() { workers := flag.Int("p", runtime.NumCPU(), "number of parallel workers") - phpunit := flag.String("bin", "vendor/bin/phpunit", "phpunit binary") + php := flag.String("php", "php", "php interpreter") + // the real PHP entry script (runs cross-platform via `php`); vendor/bin/phpunit + // is a shell/batch proxy on Windows and cannot be passed to php directly. + phpunit := flag.String("bin", "vendor/phpunit/phpunit/phpunit", "phpunit entry script") flag.Parse() dirs := flag.Args() @@ -47,7 +50,7 @@ func main() { chunks := balance(classes, *workers) start := time.Now() - failed := run(chunks, *phpunit, *workers) + failed := run(chunks, *php, *phpunit, *workers) elapsed := time.Since(start) totalFixtures := 0 @@ -126,7 +129,7 @@ func balance(classes []testClass, n int) [][]testClass { return out } -func run(chunks [][]testClass, bin string, workers int) int { +func run(chunks [][]testClass, php, phpunit string, workers int) int { sem := make(chan struct{}, workers) var wg sync.WaitGroup var mu sync.Mutex @@ -139,24 +142,28 @@ func run(chunks [][]testClass, bin string, workers int) int { sem <- struct{}{} defer func() { <-sem }() - // Each worker gets its own TMPDIR so Rector's file cache + // Each worker gets its own temp dir so Rector's file cache // (sys_get_temp_dir()/rector_cached_files) and the fixture temp - // dumper never race across processes. + // dumper never race across processes. sys_get_temp_dir() reads + // TMPDIR on Linux/macOS and TMP/TEMP on Windows, so set all three. tmp := filepath.Join(os.TempDir(), fmt.Sprintf("fast-phpunit-%d", idx)) _ = os.MkdirAll(tmp, 0o755) defer os.RemoveAll(tmp) - args := make([]string, 0, len(chunk)) + // invoke via `php ` so it works uniformly on Windows, + // where vendor/bin/phpunit is not directly executable. + args := make([]string, 0, len(chunk)+1) + args = append(args, phpunit) for _, c := range chunk { args = append(args, c.path) } - cmd := exec.Command(bin, args...) - cmd.Env = append(os.Environ(), "TMPDIR="+tmp) + cmd := exec.Command(php, args...) + cmd.Env = append(os.Environ(), "TMPDIR="+tmp, "TMP="+tmp, "TEMP="+tmp) out, err := cmd.CombinedOutput() if err != nil { mu.Lock() failed++ - fmt.Printf("chunk FAILED (%d classes):\n%s\n", len(chunk), tail(string(out), 15)) + fmt.Printf("chunk FAILED (%d classes): %v\n%s\n", len(chunk), err, tail(string(out), 15)) mu.Unlock() } }(idx, chunk) From 16b3e2705c4ac52dd5ce20294bab7ab7422e89b2 Mon Sep 17 00:00:00 2001 From: Tomas Votruba Date: Fri, 14 Aug 2026 11:19:59 +0200 Subject: [PATCH 3/4] CI: add .exe suffix to Go build output on Windows go build -o writes the exact name (no auto .exe), so the run step could not find the binary on Windows. Suffix both build and run consistently. --- .github/workflows/tests.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 9a97d37c1db..69d05b1b111 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -68,9 +68,9 @@ jobs: with: go-version: 'stable' - - run: go build -o fast-phpunit . + # `-o` writes the exact name, so add .exe on Windows ourselves + - run: go build -o fast-phpunit${{ runner.os == 'Windows' && '.exe' || '' }} . working-directory: utils-tests-runner - # run from repo root so test-dir discovery and phpunit.xml resolve; - # Go appends .exe to the output name on Windows + # run from repo root so test-dir discovery and phpunit.xml resolve - run: ./utils-tests-runner/fast-phpunit${{ runner.os == 'Windows' && '.exe' || '' }} From dbcacc54fc4dc0ce81399dcbbabcc52bc679faa7 Mon Sep 17 00:00:00 2001 From: Tomas Votruba Date: Fri, 14 Aug 2026 11:24:30 +0200 Subject: [PATCH 4/4] README: add CI-measured cross-platform numbers (Windows 116s -> 67s) --- .github/workflows/benchmark_phpunit.yaml | 56 ++++++++++++++++++++++++ .github/workflows/tests.yaml | 35 --------------- utils-tests-runner/README.md | 16 ++++++- 3 files changed, 70 insertions(+), 37 deletions(-) create mode 100644 .github/workflows/benchmark_phpunit.yaml diff --git a/.github/workflows/benchmark_phpunit.yaml b/.github/workflows/benchmark_phpunit.yaml new file mode 100644 index 00000000000..c1bab0366ca --- /dev/null +++ b/.github/workflows/benchmark_phpunit.yaml @@ -0,0 +1,56 @@ +name: Benchmark PHPUnit + +# A/B wall-time benchmark: serial phpunit vs the Go parallel runner. +# Scheduled only (not on pull requests); runs every 2 hours on ubuntu + windows. +# Reminder: cron fires only from the default branch, so this starts running +# once the file is on `main`. +on: + schedule: + - cron: '0 */2 * * *' + workflow_dispatch: null + +env: + COMPOSER_ROOT_VERSION: "dev-main" + +jobs: + benchmark: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest] + php-versions: ['8.4', '8.5'] + + runs-on: ${{ matrix.os }} + timeout-minutes: 15 + + name: benchmark (${{ matrix.os }}) + steps: + - uses: actions/checkout@v5 + + - + uses: shivammathur/setup-php@v2 + with: + php-version: ${{ matrix.php-versions }} + coverage: none + ini-values: zend.assertions=1 + + - uses: "ramsey/composer-install@v4" + + - uses: actions/setup-go@v5 + with: + go-version: 'stable' + + - name: Serial phpunit + shell: bash + run: | + printf '| platform | php | mode | wall time |\n| --- | --- | --- | --- |\n' >> "$GITHUB_STEP_SUMMARY" + start=$SECONDS + vendor/bin/phpunit + echo "| ${{ matrix.os }} | ${{ matrix.php-versions }} | serial | $((SECONDS - start))s |" >> "$GITHUB_STEP_SUMMARY" + + - name: Go parallel runner + shell: bash + run: | + start=$SECONDS + go run ./utils-tests-runner/main.go + echo "| ${{ matrix.os }} | ${{ matrix.php-versions }} | go-runner | $((SECONDS - start))s |" >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 69d05b1b111..a3378b794f7 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -39,38 +39,3 @@ jobs: - uses: "ramsey/composer-install@v4" - run: vendor/bin/phpunit --colors - - # proof of concept: same suite via the Go parallel runner, for an A/B time comparison - fast_tests: - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, windows-latest] - php-versions: ['8.4'] - - runs-on: ${{ matrix.os }} - timeout-minutes: 4 - - name: PHP ${{ matrix.php-versions }} fast-phpunit (${{ matrix.os }}) - steps: - - uses: actions/checkout@v5 - - - - uses: shivammathur/setup-php@v2 - with: - php-version: ${{ matrix.php-versions }} - coverage: none - ini-values: zend.assertions=1 - - - uses: "ramsey/composer-install@v4" - - - uses: actions/setup-go@v5 - with: - go-version: 'stable' - - # `-o` writes the exact name, so add .exe on Windows ourselves - - run: go build -o fast-phpunit${{ runner.os == 'Windows' && '.exe' || '' }} . - working-directory: utils-tests-runner - - # run from repo root so test-dir discovery and phpunit.xml resolve - - run: ./utils-tests-runner/fast-phpunit${{ runner.os == 'Windows' && '.exe' || '' }} diff --git a/utils-tests-runner/README.md b/utils-tests-runner/README.md index fdd600b5d00..4b930c5113a 100644 --- a/utils-tests-runner/README.md +++ b/utils-tests-runner/README.md @@ -3,9 +3,21 @@ Run the PHPUnit suite **~4x faster** by splitting test classes across parallel workers that each boot PHP **once** and run many classes in a single process. -## Speed — measured on this suite +## Speed — CI, per platform (GitHub runners, 4 vCPU, full suite) -Full suite: **685 test classes / 5833 fixtures**, 24-core host. +Serial `vendor/bin/phpunit` vs this runner, measured on the CI run step: + +| Platform | Serial | Go runner | Speedup | +| --- | --- | --- | --- | +| ubuntu-latest | 31s | **16s** | ~1.9x | +| windows-latest | **116s** | **67s** | **~1.7x** | + +Windows is the slow platform (~3.7x slower than Ubuntu serial); the runner cuts +~49s off it. Both run the full suite and pass. + +## Speed — local (24-core host, full suite) + +Full suite: **685 test classes / 5833 fixtures**. | Mode | Wall time | vs serial | | --- | --- | --- |