From b45102fe4fbd85e1250239ba64bcbce7fb57c349 Mon Sep 17 00:00:00 2001 From: Sander Muller Date: Thu, 17 Sep 2026 12:45:24 +0200 Subject: [PATCH 1/2] Derive the parallel worker count from the file count A warm run of a few dozen changed files uses one worker today, on any machine. jobSize 20 and minimumNumberOfJobsPerProcess 2 together ask for 40 files before a second worker is allowed, so an edit-and-rerun cycle is single threaded while the other cores idle. Behind featureToggles.adaptiveParallelWorkerCount, off by default and on in bleedingEdge, the worker count comes from the file count instead: workers = clamp(round(0.5 * sqrt(files)), files >= 9 ? 2 : 1, cores) A worker's startup is a fixed cost, so the number of files it needs to earn its keep grows with the size of the run. sqrt(files) encodes that: about 12 files per worker at 25 files, about 40 at 400, saturating at the usable cores from roughly 800 files upward. Full runs therefore keep the schedule they have today, jobs and workers both, which the tests assert at 800 and 4524 files. The job count follows the worker count, because the spawn loop stops when the queue runs dry and a worker without a job of its own never starts. Warm runs of phpstan-src, one cache snapshot per arm, 3 rounds, medians, with a duplicate control arm (0.5 to 3.7 pct): files default adaptive wall CPU 9 1w 4.32s 2w 4.19s -3.0% +25% 25 1w 4.82s 3w 4.41s -8.5% +44% 50 1w 6.93s 4w 5.11s -26.3% +50% 100 2w 7.56s 5w 5.96s -21.2% +35% 200 5w 8.21s 7w 7.50s -8.6% +14% The trade is CPU for latency, so it is opt-in. Co-Authored-By: Claude Opus 5 (1M context) --- conf/bleedingEdge.neon | 1 + conf/config.neon | 1 + conf/parametersSchema.neon | 1 + src/Parallel/Scheduler.php | 58 ++++++++++++++++++++++-- tests/PHPStan/Parallel/SchedulerTest.php | 53 ++++++++++++++++++++++ 5 files changed, 110 insertions(+), 4 deletions(-) diff --git a/conf/bleedingEdge.neon b/conf/bleedingEdge.neon index 3d9d45eb107..318b968315e 100644 --- a/conf/bleedingEdge.neon +++ b/conf/bleedingEdge.neon @@ -1,5 +1,6 @@ parameters: featureToggles: + adaptiveParallelWorkerCount: true bleedingEdge: true checkNonStringableDynamicAccess: true checkParameterCastableToNumberFunctions: true diff --git a/conf/config.neon b/conf/config.neon index 52f5dd061ec..605946bd343 100644 --- a/conf/config.neon +++ b/conf/config.neon @@ -26,6 +26,7 @@ parameters: tooWideImplicitThrowType: false throwTypeCovariance: false featureToggles: + adaptiveParallelWorkerCount: false bleedingEdge: false checkNonStringableDynamicAccess: false checkParameterCastableToNumberFunctions: false diff --git a/conf/parametersSchema.neon b/conf/parametersSchema.neon index fd9dfd4d5d6..b3a244d490f 100644 --- a/conf/parametersSchema.neon +++ b/conf/parametersSchema.neon @@ -30,6 +30,7 @@ parametersSchema: ]) featureToggles: structure([ bleedingEdge: bool(), + adaptiveParallelWorkerCount: bool(), checkNonStringableDynamicAccess: bool(), checkParameterCastableToNumberFunctions: bool(), skipCheckGenericClasses: listOf(string()), diff --git a/src/Parallel/Scheduler.php b/src/Parallel/Scheduler.php index 604792a3406..6910630b5c4 100644 --- a/src/Parallel/Scheduler.php +++ b/src/Parallel/Scheduler.php @@ -12,7 +12,9 @@ use function floor; use function max; use function min; +use function round; use function sprintf; +use function sqrt; use function usort; #[AutowiredService] @@ -29,6 +31,20 @@ final class Scheduler implements DiagnoseExtension */ private const AUTO_PROCESSES_LIMIT = 20; + /** + * Workers per square root of the file count, when the adaptive strategy is on. + * + * A worker's startup is a fixed cost, so the number of files it needs to earn + * its keep grows with the size of the run. Deriving the worker count from + * sqrt(files) encodes that: about 12 files per worker at 25 files, about 40 at + * 400, saturating at the usable core count from roughly 800 files upward - so + * full runs are scheduled exactly as before and only small ones change. + */ + private const ADAPTIVE_WORKERS_PER_SQRT_FILE = 0.5; + + /** Below this many files a second worker does not pay for its own startup. */ + private const ADAPTIVE_SECOND_WORKER_FILE_THRESHOLD = 9; + /** @var array{int, int, int, int, string}|null */ private ?array $storedData = null; @@ -44,6 +60,8 @@ public function __construct( private int|string $maximumNumberOfProcesses, #[AutowiredParameter(ref: '%parallel.minimumNumberOfJobsPerProcess%')] private int $minimumNumberOfJobsPerProcess, + #[AutowiredParameter(ref: '%featureToggles.adaptiveParallelWorkerCount%')] + private bool $adaptiveParallelWorkerCount = false, ) { } @@ -70,6 +88,19 @@ public function scheduleWork( usort($files, static fn (string $a, string $b): int => $fileSizes[$b] <=> $fileSizes[$a]); $numberOfJobs = (int) ceil(count($files) / $this->jobSize); + + $desiredNumberOfProcesses = null; + if ($this->adaptiveParallelWorkerCount) { + $desiredNumberOfProcesses = $this->resolveDesiredNumberOfProcesses(count($files), $cpuCores); + + // the spawn loop stops when the queue runs dry, so a worker without a + // job of its own never starts - chunk finely enough to feed them all + $numberOfJobs = min( + count($files), + max($numberOfJobs, $desiredNumberOfProcesses * $this->minimumNumberOfJobsPerProcess), + ); + } + $stripedJobs = []; foreach ($files as $i => $file) { $stripedJobs[$i % $numberOfJobs][] = $file; @@ -83,10 +114,12 @@ public function scheduleWork( unset($stripedJob); $jobs = array_values($stripedJobs); - $numberOfProcesses = min( - max((int) floor(count($jobs) / $this->minimumNumberOfJobsPerProcess), 1), - $cpuCores, - ); + $numberOfProcesses = $desiredNumberOfProcesses !== null + ? min($desiredNumberOfProcesses, count($jobs), $cpuCores) + : min( + max((int) floor(count($jobs) / $this->minimumNumberOfJobsPerProcess), 1), + $cpuCores, + ); [$maximumNumberOfProcesses, $decision] = $this->resolveMaximumNumberOfProcesses($cpuCores); $usedNumberOfProcesses = min($numberOfProcesses, $maximumNumberOfProcesses); @@ -95,6 +128,23 @@ public function scheduleWork( return new Schedule($usedNumberOfProcesses, $jobs); } + /** + * How many workers the file count justifies, before the configured maximum applies. + * + * @return positive-int + */ + private function resolveDesiredNumberOfProcesses(int $numberOfFiles, int $cpuCores): int + { + if ($numberOfFiles < 1) { + return 1; + } + + $floor = $numberOfFiles >= self::ADAPTIVE_SECOND_WORKER_FILE_THRESHOLD ? 2 : 1; + $fromFileCount = (int) round(self::ADAPTIVE_WORKERS_PER_SQRT_FILE * sqrt($numberOfFiles)); + + return max(1, min(max($floor, $fromFileCount), $cpuCores, $numberOfFiles)); + } + /** * How many workers may run at once, and a human-readable account of why - which * `diagnose` prints, because a user who thinks the number is wrong needs to see diff --git a/tests/PHPStan/Parallel/SchedulerTest.php b/tests/PHPStan/Parallel/SchedulerTest.php index de2b132980a..5cdb148886c 100644 --- a/tests/PHPStan/Parallel/SchedulerTest.php +++ b/tests/PHPStan/Parallel/SchedulerTest.php @@ -195,6 +195,59 @@ public function testAutoIsStillCappedByTheJobCount(): void $this->assertSame(1, $schedule->getNumberOfProcesses()); } + public function testAdaptiveWorkerCountLeavesFullRunsAlone(): void + { + // from ~800 files the sqrt rule saturates at the usable cores, so a full + // run is scheduled exactly as it is without the toggle + foreach ([800, 4524] as $numberOfFiles) { + $files = array_fill(0, $numberOfFiles, 'file.php'); + $legacy = (new Scheduler(20, Scheduler::AUTO, 2))->scheduleWork(14, $files, static fn (string $file): int => 0); + $adaptive = (new Scheduler(20, Scheduler::AUTO, 2, true))->scheduleWork(14, $files, static fn (string $file): int => 0); + + $this->assertSame($legacy->getNumberOfProcesses(), $adaptive->getNumberOfProcesses()); + $this->assertSame($legacy->getJobs(), $adaptive->getJobs()); + } + } + + public function testAdaptiveWorkerCountParallelisesSmallRuns(): void + { + // 50 files is one worker today, because jobSize 20 and 2 jobs per process + // ask for 40 files before a second worker is allowed + $files = array_fill(0, 50, 'file.php'); + $callback = static fn (string $file): int => 0; + + $this->assertSame(1, (new Scheduler(20, Scheduler::AUTO, 2))->scheduleWork(14, $files, $callback)->getNumberOfProcesses()); + $this->assertSame(4, (new Scheduler(20, Scheduler::AUTO, 2, true))->scheduleWork(14, $files, $callback)->getNumberOfProcesses()); + } + + public function testAdaptiveWorkerCountKeepsTinyRunsSerial(): void + { + // below the threshold a second worker does not pay for its own startup + $callback = static fn (string $file): int => 0; + $scheduler = new Scheduler(20, Scheduler::AUTO, 2, true); + + $this->assertSame(1, $scheduler->scheduleWork(14, array_fill(0, 1, 'file.php'), $callback)->getNumberOfProcesses()); + $this->assertSame(1, $scheduler->scheduleWork(14, array_fill(0, 8, 'file.php'), $callback)->getNumberOfProcesses()); + $this->assertSame(2, $scheduler->scheduleWork(14, array_fill(0, 9, 'file.php'), $callback)->getNumberOfProcesses()); + } + + public function testAdaptiveWorkerCountNeverExceedsTheJobCount(): void + { + // a worker with no job never starts, so the schedule must not claim one + $callback = static fn (string $file): int => 0; + $schedule = (new Scheduler(20, Scheduler::AUTO, 2, true))->scheduleWork(14, array_fill(0, 3, 'file.php'), $callback); + + $this->assertLessThanOrEqual(count($schedule->getJobs()), $schedule->getNumberOfProcesses()); + } + + public function testAdaptiveWorkerCountStillRespectsAnExplicitLimit(): void + { + $callback = static fn (string $file): int => 0; + $schedule = (new Scheduler(20, 2, 2, true))->scheduleWork(14, array_fill(0, 200, 'file.php'), $callback); + + $this->assertSame(2, $schedule->getNumberOfProcesses()); + } + public function testAnExplicitLimitStillWins(): void { $scheduler = new Scheduler(1, 20, 1); From 372a15efdd3cb54773bb8033ba7e1eb41250dfa7 Mon Sep 17 00:00:00 2001 From: Sander Muller Date: Thu, 17 Sep 2026 17:12:37 +0200 Subject: [PATCH 2/2] Never schedule fewer workers than the file count already justifies sqrt(files) dips below the existing job-count formula between roughly 400 and 800 files, so the rule quietly took workers away from large runs. On 600 warm files it scheduled 12 where the default schedules 14, and measured 13.2% slower against a 2.2% control. The rule exists to stop small runs being starved, never to reduce a large one. Take the maximum of the two, so the adaptive count can only ever raise parallelism. Small runs are unchanged: 9, 25, 50, 100 and 200 files still schedule 2, 3, 4, 5 and 7 workers. The test asserts the invariant across 13 file counts rather than trusting a table, because the dip only shows up in a band neither end of the range covers. Reported by @staabm on the pull request. Co-Authored-By: Claude Opus 5 (1M context) --- src/Parallel/Scheduler.php | 11 ++++++++--- tests/PHPStan/Parallel/SchedulerTest.php | 19 +++++++++++++++++++ 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/src/Parallel/Scheduler.php b/src/Parallel/Scheduler.php index 6910630b5c4..6b23c0f621e 100644 --- a/src/Parallel/Scheduler.php +++ b/src/Parallel/Scheduler.php @@ -91,7 +91,7 @@ public function scheduleWork( $desiredNumberOfProcesses = null; if ($this->adaptiveParallelWorkerCount) { - $desiredNumberOfProcesses = $this->resolveDesiredNumberOfProcesses(count($files), $cpuCores); + $desiredNumberOfProcesses = $this->resolveDesiredNumberOfProcesses(count($files), $numberOfJobs, $cpuCores); // the spawn loop stops when the queue runs dry, so a worker without a // job of its own never starts - chunk finely enough to feed them all @@ -133,7 +133,7 @@ public function scheduleWork( * * @return positive-int */ - private function resolveDesiredNumberOfProcesses(int $numberOfFiles, int $cpuCores): int + private function resolveDesiredNumberOfProcesses(int $numberOfFiles, int $numberOfJobs, int $cpuCores): int { if ($numberOfFiles < 1) { return 1; @@ -142,7 +142,12 @@ private function resolveDesiredNumberOfProcesses(int $numberOfFiles, int $cpuCor $floor = $numberOfFiles >= self::ADAPTIVE_SECOND_WORKER_FILE_THRESHOLD ? 2 : 1; $fromFileCount = (int) round(self::ADAPTIVE_WORKERS_PER_SQRT_FILE * sqrt($numberOfFiles)); - return max(1, min(max($floor, $fromFileCount), $cpuCores, $numberOfFiles)); + // never below what the job count alone already justifies: the point is to stop + // small runs being starved, not to take workers away from large ones, and + // sqrt() dips under the existing formula between roughly 400 and 800 files + $fromJobCount = max((int) floor($numberOfJobs / $this->minimumNumberOfJobsPerProcess), 1); + + return max(1, min(max($floor, $fromFileCount, $fromJobCount), $cpuCores, $numberOfFiles)); } /** diff --git a/tests/PHPStan/Parallel/SchedulerTest.php b/tests/PHPStan/Parallel/SchedulerTest.php index 5cdb148886c..9b2c9bdfa2e 100644 --- a/tests/PHPStan/Parallel/SchedulerTest.php +++ b/tests/PHPStan/Parallel/SchedulerTest.php @@ -231,6 +231,25 @@ public function testAdaptiveWorkerCountKeepsTinyRunsSerial(): void $this->assertSame(2, $scheduler->scheduleWork(14, array_fill(0, 9, 'file.php'), $callback)->getNumberOfProcesses()); } + public function testAdaptiveWorkerCountIsNeverBelowTheDefault(): void + { + // the rule exists to stop small runs being starved, never to take workers away + // from large ones - sqrt() alone dips under the existing formula between roughly + // 400 and 800 files, which measured 13% slower at 600 + $callback = static fn (string $file): int => 0; + foreach ([1, 5, 9, 25, 50, 100, 200, 300, 400, 600, 800, 1424, 4524] as $numberOfFiles) { + $files = array_fill(0, $numberOfFiles, 'file.php'); + $legacy = (new Scheduler(20, Scheduler::AUTO, 2))->scheduleWork(14, $files, $callback); + $adaptive = (new Scheduler(20, Scheduler::AUTO, 2, true))->scheduleWork(14, $files, $callback); + + $this->assertGreaterThanOrEqual( + $legacy->getNumberOfProcesses(), + $adaptive->getNumberOfProcesses(), + sprintf('%d files', $numberOfFiles), + ); + } + } + public function testAdaptiveWorkerCountNeverExceedsTheJobCount(): void { // a worker with no job never starts, so the schedule must not claim one