From c0ff210895f5c419adf873ba3849f6dc01639bc6 Mon Sep 17 00:00:00 2001 From: Alain Schlesser Date: Mon, 2 Nov 2026 03:41:56 +0000 Subject: [PATCH 1/3] Collect code coverage in parallel workers and shards Co-Authored-By: Claude Opus 5.5 --- bin/run-behat-tests | 10 ++-- src/Context/FeatureContext.php | 33 ++++++++++++- utils/behat-parallel.php | 85 +++++++++++++++++++++++++++++++++- utils/generate-coverage.php | 5 +- 4 files changed, 123 insertions(+), 10 deletions(-) diff --git a/bin/run-behat-tests b/bin/run-behat-tests index ff23541d5..ab068ce41 100755 --- a/bin/run-behat-tests +++ b/bin/run-behat-tests @@ -341,9 +341,6 @@ if [ -n "${PARALLEL}" ] && [ "${PARALLEL}" != "1" ]; then if ! is_numeric "${PARALLEL}"; then echo "Warning: WP_CLI_TEST_PARALLEL is not a number of workers, running the scenarios one after the other." >&2 PARALLEL="" - elif [[ "${WP_CLI_TEST_COVERAGE}" == "true" ]]; then - echo "Code coverage is collected, running the scenarios one after the other." >&2 - PARALLEL="" elif [ "${WP_CLI_TEST_DBTYPE-}" != "sqlite" ] && ! ${DB_CLIENT} ${HOST_STRING} --user="${USER}" ${PASSWORD_ARG} --execute="CREATE DATABASE IF NOT EXISTS \`${DBNAME}_1\`; DROP DATABASE \`${DBNAME}_1\`;" 2>/dev/null; then echo "Warning: The test user cannot create the databases of parallel workers, running the scenarios one after the other." >&2 echo "Run 'composer prepare-tests' once to allow it." >&2 @@ -351,8 +348,11 @@ if [ -n "${PARALLEL}" ] && [ "${PARALLEL}" != "1" ]; then fi fi -if [ -n "${PARALLEL}" ] && [ "${PARALLEL}" != "1" ]; then - exec php "$WP_CLI_TESTS_ROOT"/utils/behat-parallel.php "${PARALLEL}" --snippets-for="WP_CLI\Tests\Context\FeatureContext" "${FORMAT_ARGS[@]}" "$BEHAT_TAGS" --strict "${BEHAT_EXTRA_ARGS[@]}" -- "$@" +# WP_CLI_TEST_SHARD set to "/", for example "2/3", runs only that share of the +# scenarios, so that the suite can be split across several machines. Every share is picked +# from the same cost-balanced distribution, so together they cover every scenario exactly once. +if [ -n "${WP_CLI_TEST_SHARD-}" ] || { [ -n "${PARALLEL}" ] && [ "${PARALLEL}" != "1" ]; }; then + exec php "$WP_CLI_TESTS_ROOT"/utils/behat-parallel.php "${PARALLEL:-1}" --snippets-for="WP_CLI\Tests\Context\FeatureContext" "${FORMAT_ARGS[@]}" "$BEHAT_TAGS" --strict "${BEHAT_EXTRA_ARGS[@]}" -- "$@" fi vendor/bin/behat --snippets-for="WP_CLI\Tests\Context\FeatureContext" "${FORMAT_ARGS[@]}" "$BEHAT_TAGS" --strict "${BEHAT_EXTRA_ARGS[@]}" "$@" diff --git a/src/Context/FeatureContext.php b/src/Context/FeatureContext.php index 1692abc6d..3e6116656 100644 --- a/src/Context/FeatureContext.php +++ b/src/Context/FeatureContext.php @@ -320,7 +320,18 @@ class_exists( Selector::class ) ? ( new Selector() )->forLineCoverage( $filter ) $filter ); - foreach ( new DirectoryIterator( self::$behat_run_dir . '/build/logs' ) as $file ) { + // Parallel workers each merge the coverage they collected into a report of their own, + // as they would otherwise merge, and remove, each other's files while those are written. + $coverage_id = self::get_coverage_id(); + $logs_dir = self::$behat_run_dir . '/build/logs'; + $source_dir = '' !== $coverage_id ? "{$logs_dir}/behat-{$coverage_id}" : $logs_dir; + $report = '' !== $coverage_id ? "{$logs_dir}/behat-coverage-{$coverage_id}.xml" : "{$logs_dir}/behat-coverage.xml"; + + if ( ! is_dir( $source_dir ) ) { + return; + } + + foreach ( new DirectoryIterator( $source_dir ) as $file ) { if ( ! $file->isFile() || 'cov' !== $file->getExtension() ) { continue; } @@ -329,7 +340,20 @@ class_exists( Selector::class ) ? ( new Selector() )->forLineCoverage( $filter ) unlink( $file->getPathname() ); } - ( new Clover() )->process( $coverage, self::$behat_run_dir . '/build/logs/behat-coverage.xml' ); + ( new Clover() )->process( $coverage, $report ); + + if ( $source_dir !== $logs_dir ) { + @rmdir( $source_dir ); + } + } + + /** + * Get the identifier of the parallel worker the code coverage is collected for, if any. + * + * @return string + */ + private static function get_coverage_id() { + return (string) preg_replace( '/[^A-Za-z0-9_-]+/', '-', (string) getenv( 'WP_CLI_TEST_COVERAGE_ID' ) ); } /** @@ -506,6 +530,11 @@ private static function get_process_env_variables(): array { $current = getenv( 'WP_CLI_REQUIRE' ); $updated = $current ? "{$current},{$coverage_require_file}" : $coverage_require_file; $env['WP_CLI_REQUIRE'] = $updated; + + $coverage_id = self::get_coverage_id(); + if ( '' !== $coverage_id ) { + $env['WP_CLI_TEST_COVERAGE_ID'] = $coverage_id; + } } $env = array_filter( diff --git a/utils/behat-parallel.php b/utils/behat-parallel.php index 3de8b7b33..8ce7e0736 100644 --- a/utils/behat-parallel.php +++ b/utils/behat-parallel.php @@ -239,6 +239,64 @@ static function ( $a, $b ) { return $shards; } +/** + * Get the share of the scenarios to run, as set through WP_CLI_TEST_SHARD to "/". + * + * @return array{int, int}|null One-based index and total number of shards, or null to run all of them. + */ +function wp_cli_tests_shard() { + $value = trim( (string) getenv( 'WP_CLI_TEST_SHARD' ) ); + + if ( '' === $value ) { + return null; + } + + if ( ! preg_match( '#^(\d+)/(\d+)$#', $value, $matches ) || (int) $matches[1] < 1 || (int) $matches[1] > (int) $matches[2] ) { + fwrite( STDERR, "Warning: WP_CLI_TEST_SHARD is not of the form /, like 2/3, running all scenarios.\n" ); + return null; + } + + return [ (int) $matches[1], (int) $matches[2] ]; +} + +/** + * Pick one shard of the scenarios, out of a cost-balanced distribution over all shards. + * + * Scenarios that rely on shared resources are spread over the shards as well, as every shard runs + * on a machine of its own. The distribution only depends on the scenarios, so every shard computes + * the same one and together they run every scenario exactly once. + * + * @param array $scenarios + * @param int $index One-based. + * @param int $total + * @return array + */ +function wp_cli_tests_pick_shard( array $scenarios, $index, $total ) { + $order = array_keys( $scenarios ); + + usort( + $order, + static function ( $a, $b ) use ( $scenarios ) { + return ( $scenarios[ $b ]['cost'] - $scenarios[ $a ]['cost'] ) ?: ( $a - $b ); + } + ); + + $load = array_fill( 0, $total, 0 ); + $picked = []; + + foreach ( $order as $key ) { + $shard = (int) array_search( min( $load ), $load, true ); + $load[ $shard ] += $scenarios[ $key ]['cost']; + if ( $shard === $index - 1 ) { + $picked[ $key ] = $scenarios[ $key ]; + } + } + + ksort( $picked ); + + return array_values( $picked ); +} + /** * Get the Docker image of the database server the tests run against on the host, so that the * containers test against the same type and version of database server. @@ -628,7 +686,9 @@ function wp_cli_tests_docker_worker_command( $name, $php_image, $db_image, array } } -if ( ( $workers < 2 && ! $docker ) || DIRECTORY_SEPARATOR === '\\' || ( $selects_scenarios && null === $rerun_paths && ! $docker ) ) { +$shard = $selects_scenarios || null !== $rerun_paths ? null : wp_cli_tests_shard(); + +if ( ( $workers < 2 && ! $docker && null === $shard ) || DIRECTORY_SEPARATOR === '\\' || ( $selects_scenarios && null === $rerun_paths && ! $docker ) ) { wp_cli_tests_run_behat_and_exit( $all_args ); } @@ -737,10 +797,28 @@ static function ( $path ) { $scenarios = wp_cli_tests_collect_scenarios( $features_folder, $tags ); } +if ( null !== $shard ) { + $all_scenarios = count( $scenarios ); + $scenarios = wp_cli_tests_pick_shard( $scenarios, $shard[0], $shard[1] ); + printf( "Running shard %d of %d, with %d of %d scenarios.\n", $shard[0], $shard[1], count( $scenarios ), $all_scenarios ); +} + $workers = min( max( 1, $workers ), max( 1, count( $scenarios ) ) ); if ( $workers < 2 && ! $docker ) { - wp_cli_tests_run_behat_and_exit( $all_args ); + if ( null === $shard ) { + wp_cli_tests_run_behat_and_exit( $all_args ); + } + + if ( ! is_dir( $base_dir ) ) { + mkdir( $base_dir, 0777, true ); + } + $list = "{$base_dir}/shard.scenarios"; + file_put_contents( $list, implode( "\n", array_column( $scenarios, 'path' ) ) . "\n" ); + if ( count( $scenarios ) > 0 ) { + wp_cli_tests_run_behat_and_exit( array_merge( $all_args, [ $list ] ) ); + } + exit( 0 ); } // Containers do not share ports or temporary directories, so any of them can run any scenario. @@ -778,6 +856,9 @@ static function () use ( &$containers ) { $env = getenv(); + // Every worker merges the code coverage of its own scenarios into a report of its own. + $env['WP_CLI_TEST_COVERAGE_ID'] = ( null !== $shard ? "shard-{$shard[0]}-" : '' ) . "worker-{$index}"; + if ( $index > 0 ) { $env['TMPDIR'] = "{$base_dir}/worker-{$index}"; $env['WP_CLI_TEST_DBNAME'] = "{$db_name}_{$index}"; diff --git a/utils/generate-coverage.php b/utils/generate-coverage.php index 007e66ae1..e88b14a65 100644 --- a/utils/generate-coverage.php +++ b/utils/generate-coverage.php @@ -96,7 +96,10 @@ static function () use ( $coverage, $feature, $scenario, $step_line, $name, $pro $feature_suffix = preg_replace( '/[^a-z0-9]+/', '-', strtolower( $feature ) ); $scenario_suffix = preg_replace( '/[^a-z0-9]+/', '-', strtolower( $scenario ) ); $db_type = strtolower( getenv( 'WP_CLI_TEST_DBTYPE' ) ); - $destination = "$project_dir/build/logs/$feature_suffix-$scenario_suffix-$step_line-$db_type.cov"; + // Parallel workers each collect into a directory of their own, see FeatureContext::merge_coverage_reports(). + $coverage_id = preg_replace( '/[^A-Za-z0-9_-]+/', '-', (string) getenv( 'WP_CLI_TEST_COVERAGE_ID' ) ); + $coverage_dir = '' !== $coverage_id ? "$project_dir/build/logs/behat-$coverage_id" : "$project_dir/build/logs"; + $destination = "$coverage_dir/$feature_suffix-$scenario_suffix-$step_line-$db_type.cov"; $dir = dirname( $destination ); if ( ! file_exists( $dir ) ) { From 37466e9eeda8cde54cc007ec10f74f2369f82697 Mon Sep 17 00:00:00 2001 From: Alain Schlesser Date: Mon, 2 Nov 2026 04:09:36 +0000 Subject: [PATCH 2/3] Build Phars of the bundle one after the other when collecting coverage Co-Authored-By: Claude Opus 5.5 --- utils/behat-parallel.php | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/utils/behat-parallel.php b/utils/behat-parallel.php index 8ce7e0736..a3be05473 100644 --- a/utils/behat-parallel.php +++ b/utils/behat-parallel.php @@ -157,6 +157,12 @@ function wp_cli_tests_collect_scenarios( $features_folder, $tags ) { $filter = '' !== $tags ? new TagFilter( $tags ) : null; $scenarios = []; + // With code coverage, building a Phar of the bundle itself rewrites its Composer autoloader + // for the duration of the build, which would break the scenarios of the other workers. + $phar_shared = in_array( (string) getenv( 'WP_CLI_TEST_COVERAGE' ), [ 'true', '1' ], true ) + && ! file_exists( 'vendor/wp-cli/wp-cli-bundle/utils/make-phar.php' ) + && file_exists( 'utils/make-phar.php' ); + $files = glob( $features_folder . '/*.feature' ); sort( $files ); @@ -188,7 +194,8 @@ function wp_cli_tests_collect_scenarios( $features_folder, $tags ) { $scenarios[] = [ 'path' => $file . ':' . $scenario->getLine(), 'cost' => $cost, - 'shared' => (bool) preg_match( WP_CLI_TESTS_SHARED_RESOURCES, $text ), + 'shared' => (bool) preg_match( WP_CLI_TESTS_SHARED_RESOURCES, $text ) + || ( $phar_shared && preg_match( '/^a new Phar/m', $text ) ), ]; } } From e0760cf45a7cf162389bcb17cc9ded712e8490e3 Mon Sep 17 00:00:00 2001 From: Alain Schlesser Date: Mon, 2 Nov 2026 10:10:52 +0000 Subject: [PATCH 3/3] Test splitting the scenarios across shards, and document it Co-Authored-By: Claude Opus 5.5 --- README.md | 9 +- tests/tests/TestBehatParallel.php | 264 ++++++++++++++++++++++++++++++ 2 files changed, 272 insertions(+), 1 deletion(-) create mode 100644 tests/tests/TestBehatParallel.php diff --git a/README.md b/README.md index 1a34a4485..12f85d113 100644 --- a/README.md +++ b/README.md @@ -272,10 +272,17 @@ Set `WP_CLI_TEST_PARALLEL` to a number of workers to spread the scenarios over t WP_CLI_TEST_PARALLEL=4 composer behat ``` -Every worker but the first gets its own temporary directory, and with it its own home directory, caches and run directories, as well as its own test database, named after the test database with a numeric suffix (`wp_cli_test_1`, `wp_cli_test_2`, ...). Run `composer prepare-tests` once to allow the test user to create these databases. Without that permission, the scenarios are run one after the other, as they are on Windows, when collecting code coverage, or when selecting scenarios by path or name. +Every worker but the first gets its own temporary directory, and with it its own home directory, caches and run directories, as well as its own test database, named after the test database with a numeric suffix (`wp_cli_test_1`, `wp_cli_test_2`, ...). Run `composer prepare-tests` once to allow the test user to create these databases. Without that permission, the scenarios are run one after the other, as they are on Windows or when selecting scenarios by path or name. When collecting code coverage, every worker writes a report of its own to `build/logs`. Scenarios that rely on resources shared across the system, such as a fixed port or a fixed path in the system's temporary directory, are all run by the first worker, one after the other. `composer behat-rerun` re-runs exactly the scenarios that failed in the last parallel run. +Set `WP_CLI_TEST_SHARD` to `/` to run only that share of the scenarios, to split them across several machines: +```bash +WP_CLI_TEST_SHARD=2/4 WP_CLI_TEST_PARALLEL=4 composer behat +``` + +Every machine spreads the scenarios over the shards by their expected cost in the same way, so that together they run every scenario once and finish at about the same time. Selecting scenarios by path or name runs them regardless of the shard. + Set `WP_CLI_TEST_DOCKER` to run the scenarios in throwaway Docker containers instead, every worker in its own: ```bash WP_CLI_TEST_DOCKER=auto WP_CLI_TEST_PARALLEL=4 composer behat diff --git a/tests/tests/TestBehatParallel.php b/tests/tests/TestBehatParallel.php new file mode 100644 index 000000000..3d345ddb0 --- /dev/null +++ b/tests/tests/TestBehatParallel.php @@ -0,0 +1,264 @@ +markTestSkipped( 'The runner does not split the scenarios on Windows.' ); + } + + $this->temp_dir = Utils\get_temp_dir() . uniqid( 'wp-cli-test-behat-parallel-', true ); + mkdir( $this->temp_dir . '/features', 0777, true ); + file_put_contents( $this->temp_dir . '/behat.yml', "default:\n suites:\n default:\n contexts: []\n" ); + + // The runner loads the Gherkin parser and Behat from the project it runs in. + $vendor = dirname( dirname( __DIR__ ) ) . '/vendor'; + if ( ! @symlink( $vendor, $this->temp_dir . '/vendor' ) ) { // phpcs:ignore WordPress.PHP.NoSilencedErrors.Discouraged -- Skipped below. + $this->markTestSkipped( 'The vendor directory cannot be linked into the temporary project.' ); + } + } + + protected function tear_down(): void { + if ( null === $this->temp_dir ) { + parent::tear_down(); + return; + } + + if ( is_link( $this->temp_dir . '/vendor' ) ) { + unlink( $this->temp_dir . '/vendor' ); + } + $this->remove_dir( $this->get_base_dir() ); + $this->remove_dir( $this->temp_dir ); + + parent::tear_down(); + } + + public function test_shards_run_every_scenario_once(): void { + $expected = $this->write_features(); + + $picked = []; + for ( $index = 1; $index <= 3; $index++ ) { + $result = $this->run_shard( "{$index}/3" ); + $this->assertSame( 0, $result['exit_code'], $result['output'] ); + + $shard = $this->get_shard_scenarios(); + $this->assertNotEmpty( $shard, "Shard {$index} of 3 got no scenarios." ); + $this->assertStringContainsString( sprintf( 'Running shard %d of 3, with %d of %d scenarios.', $index, count( $shard ), count( $expected ) ), $result['output'] ); + + $picked = array_merge( $picked, $shard ); + } + + sort( $picked ); + $this->assertSame( $expected, $picked ); + } + + public function test_shards_are_balanced_by_cost(): void { + $this->write_feature( + 'install.feature', + [ + 'Install one' => [ 'Given a WP install' ], + 'Install two' => [ 'Given a WP install' ], + ] + ); + $this->write_feature( + 'cheap.feature', + [ + 'Cheap one' => [ 'Given an empty directory' ], + 'Cheap two' => [ 'Given an empty directory' ], + 'Cheap three' => [ 'Given an empty directory' ], + 'Cheap four' => [ 'Given an empty directory' ], + ] + ); + + // The two installations are the most expensive by far, so each shard gets one of them. + foreach ( [ '1/2', '2/2' ] as $shard ) { + $this->run_shard( $shard ); + $installs = array_filter( + $this->get_shard_scenarios(), + static function ( $path ) { + return false !== strpos( $path, 'install.feature' ); + } + ); + $this->assertCount( 1, $installs, "Shard {$shard}" ); + } + } + + public function test_shard_without_scenarios_succeeds(): void { + $this->write_feature( 'only.feature', [ 'The only one' => [ 'Given an empty directory' ] ] ); + + $result = $this->run_shard( '2/2' ); + + $this->assertSame( 0, $result['exit_code'], $result['output'] ); + $this->assertStringContainsString( 'Running shard 2 of 2, with 0 of 1 scenarios.', $result['output'] ); + $this->assertSame( [], $this->get_shard_scenarios() ); + } + + public function test_invalid_shard_runs_all_scenarios(): void { + $this->write_features(); + + $result = $this->run_shard( '3/2' ); + + $this->assertStringContainsString( 'WP_CLI_TEST_SHARD is not of the form /', $result['output'] ); + $this->assertStringNotContainsString( 'Running shard', $result['output'] ); + } + + public function test_selected_scenarios_ignore_the_shard(): void { + $this->write_features(); + + $result = $this->run_shard( '1/3', [ 'features/a.feature' ] ); + + $this->assertStringNotContainsString( 'Running shard', $result['output'] ); + } + + /** + * Write feature files with a mix of scenarios, including an outline and one using shared resources. + * + * @return string[] Paths of all scenarios, sorted. + */ + private function write_features(): array { + $this->write_feature( + 'a.feature', + [ + 'First' => [ 'Given a WP install', 'When I run `wp option get home`' ], + 'Second' => [ 'Given an empty directory' ], + 'Server' => [ 'Given a WP install', 'And a PHP built-in web server' ], + ] + ); + $this->write_feature( + 'b.feature', + [ + 'Third' => [ 'Given an empty directory', 'When I run `wp cli version`' ], + 'Fourth' => [ 'Given a WP install with Composer' ], + ], + " Scenario Outline: Fifth\n Given an empty directory\n When I run `wp `\n\n Examples:\n | command |\n | cli info |\n | cli version |\n" + ); + $this->write_feature( + 'c.feature', + [ + 'Sixth' => [ 'Given an empty directory' ], + 'Seventh' => [ 'Given a WP install', 'When I run `wp plugin list`' ], + ] + ); + + $paths = []; + foreach ( glob( $this->temp_dir . '/features/*.feature' ) as $file ) { + $lines = file( $file ); + foreach ( $lines as $number => $line ) { + if ( preg_match( '/^\s*Scenario( Outline)?:/', $line ) ) { + $paths[] = 'features/' . basename( $file ) . ':' . ( $number + 1 ); + } + } + } + sort( $paths ); + + return $paths; + } + + /** + * @param string $name File name. + * @param array $scenarios Steps by scenario title. + * @param string $extra Additional Gherkin to append. + */ + private function write_feature( $name, array $scenarios, $extra = '' ): void { + $contents = "Feature: {$name}\n"; + foreach ( $scenarios as $title => $steps ) { + $contents .= "\n Scenario: {$title}\n " . implode( "\n ", $steps ) . "\n"; + } + if ( '' !== $extra ) { + $contents .= "\n" . $extra; + } + + file_put_contents( $this->temp_dir . '/features/' . $name, $contents ); + } + + /** + * Run the runner with a single worker, as run-behat-tests does for a shard. + * + * @param string $shard Value of WP_CLI_TEST_SHARD. + * @param string[] $user_args Arguments passed to run-behat-tests. + * @return array{output: string, exit_code: int} + */ + private function run_shard( $shard, array $user_args = [] ): array { + $script = dirname( dirname( __DIR__ ) ) . '/utils/behat-parallel.php'; + $command = array_merge( [ PHP_BINARY, $script, '1', '--dry-run', '--format=progress', '--' ], $user_args ); + + $env = array_merge( + getenv(), + [ + 'WP_CLI_TEST_SHARD' => $shard, + 'WP_CLI_TEST_DOCKER' => '0', + 'BEHAT_FEATURES_FOLDER' => 'features', + ] + ); + unset( $env['WP_CLI_TEST_COVERAGE'] ); + + $process = proc_open( + implode( ' ', array_map( 'escapeshellarg', $command ) ) . ' 2>&1', + [ 1 => [ 'pipe', 'w' ] ], + $pipes, + $this->temp_dir, + $env + ); + $this->assertIsResource( $process ); + + $output = (string) stream_get_contents( $pipes[1] ); + fclose( $pipes[1] ); + + return [ + 'output' => $output, + 'exit_code' => proc_close( $process ), + ]; + } + + /** + * Get the scenarios the last run handed to Behat for its shard. + * + * @return string[] + */ + private function get_shard_scenarios(): array { + $list = $this->get_base_dir() . '/shard.scenarios'; + $this->assertFileExists( $list ); + + return array_values( array_filter( explode( "\n", (string) file_get_contents( $list ) ) ) ); + } + + /** + * Directory the runner keeps its state in, which depends on the project directory. + */ + private function get_base_dir(): string { + return sys_get_temp_dir() . '/wp-cli-test-parallel-' . substr( md5( (string) realpath( $this->temp_dir ) ), 0, 8 ); + } + + private function remove_dir( string $dir ): void { + if ( ! is_dir( $dir ) ) { + return; + } + $items = new \RecursiveIteratorIterator( + new \RecursiveDirectoryIterator( $dir, \FilesystemIterator::SKIP_DOTS ), + \RecursiveIteratorIterator::CHILD_FIRST + ); + foreach ( $items as $item ) { + $item->isDir() && ! $item->isLink() ? rmdir( (string) $item ) : unlink( (string) $item ); + } + rmdir( $dir ); + } +}