diff --git a/README.md b/README.md index 1a34a448..12f85d11 100644 --- a/README.md +++ b/README.md @@ -272,10 +272,17 @@ Set `WP_CLI_TEST_PARALLEL` to a number of workers to spread the scenarios over t WP_CLI_TEST_PARALLEL=4 composer behat ``` -Every worker but the first gets its own temporary directory, and with it its own home directory, caches and run directories, as well as its own test database, named after the test database with a numeric suffix (`wp_cli_test_1`, `wp_cli_test_2`, ...). Run `composer prepare-tests` once to allow the test user to create these databases. Without that permission, the scenarios are run one after the other, as they are on Windows, when collecting code coverage, or when selecting scenarios by path or name. +Every worker but the first gets its own temporary directory, and with it its own home directory, caches and run directories, as well as its own test database, named after the test database with a numeric suffix (`wp_cli_test_1`, `wp_cli_test_2`, ...). Run `composer prepare-tests` once to allow the test user to create these databases. Without that permission, the scenarios are run one after the other, as they are on Windows or when selecting scenarios by path or name. When collecting code coverage, every worker writes a report of its own to `build/logs`. Scenarios that rely on resources shared across the system, such as a fixed port or a fixed path in the system's temporary directory, are all run by the first worker, one after the other. `composer behat-rerun` re-runs exactly the scenarios that failed in the last parallel run. +Set `WP_CLI_TEST_SHARD` to `/` to run only that share of the scenarios, to split them across several machines: +```bash +WP_CLI_TEST_SHARD=2/4 WP_CLI_TEST_PARALLEL=4 composer behat +``` + +Every machine spreads the scenarios over the shards by their expected cost in the same way, so that together they run every scenario once and finish at about the same time. Selecting scenarios by path or name runs them regardless of the shard. + Set `WP_CLI_TEST_DOCKER` to run the scenarios in throwaway Docker containers instead, every worker in its own: ```bash WP_CLI_TEST_DOCKER=auto WP_CLI_TEST_PARALLEL=4 composer behat diff --git a/bin/run-behat-tests b/bin/run-behat-tests index ff23541d..ab068ce4 100755 --- a/bin/run-behat-tests +++ b/bin/run-behat-tests @@ -341,9 +341,6 @@ if [ -n "${PARALLEL}" ] && [ "${PARALLEL}" != "1" ]; then if ! is_numeric "${PARALLEL}"; then echo "Warning: WP_CLI_TEST_PARALLEL is not a number of workers, running the scenarios one after the other." >&2 PARALLEL="" - elif [[ "${WP_CLI_TEST_COVERAGE}" == "true" ]]; then - echo "Code coverage is collected, running the scenarios one after the other." >&2 - PARALLEL="" elif [ "${WP_CLI_TEST_DBTYPE-}" != "sqlite" ] && ! ${DB_CLIENT} ${HOST_STRING} --user="${USER}" ${PASSWORD_ARG} --execute="CREATE DATABASE IF NOT EXISTS \`${DBNAME}_1\`; DROP DATABASE \`${DBNAME}_1\`;" 2>/dev/null; then echo "Warning: The test user cannot create the databases of parallel workers, running the scenarios one after the other." >&2 echo "Run 'composer prepare-tests' once to allow it." >&2 @@ -351,8 +348,11 @@ if [ -n "${PARALLEL}" ] && [ "${PARALLEL}" != "1" ]; then fi fi -if [ -n "${PARALLEL}" ] && [ "${PARALLEL}" != "1" ]; then - exec php "$WP_CLI_TESTS_ROOT"/utils/behat-parallel.php "${PARALLEL}" --snippets-for="WP_CLI\Tests\Context\FeatureContext" "${FORMAT_ARGS[@]}" "$BEHAT_TAGS" --strict "${BEHAT_EXTRA_ARGS[@]}" -- "$@" +# WP_CLI_TEST_SHARD set to "/", for example "2/3", runs only that share of the +# scenarios, so that the suite can be split across several machines. Every share is picked +# from the same cost-balanced distribution, so together they cover every scenario exactly once. +if [ -n "${WP_CLI_TEST_SHARD-}" ] || { [ -n "${PARALLEL}" ] && [ "${PARALLEL}" != "1" ]; }; then + exec php "$WP_CLI_TESTS_ROOT"/utils/behat-parallel.php "${PARALLEL:-1}" --snippets-for="WP_CLI\Tests\Context\FeatureContext" "${FORMAT_ARGS[@]}" "$BEHAT_TAGS" --strict "${BEHAT_EXTRA_ARGS[@]}" -- "$@" fi vendor/bin/behat --snippets-for="WP_CLI\Tests\Context\FeatureContext" "${FORMAT_ARGS[@]}" "$BEHAT_TAGS" --strict "${BEHAT_EXTRA_ARGS[@]}" "$@" diff --git a/src/Context/FeatureContext.php b/src/Context/FeatureContext.php index 1692abc6..3e611665 100644 --- a/src/Context/FeatureContext.php +++ b/src/Context/FeatureContext.php @@ -320,7 +320,18 @@ class_exists( Selector::class ) ? ( new Selector() )->forLineCoverage( $filter ) $filter ); - foreach ( new DirectoryIterator( self::$behat_run_dir . '/build/logs' ) as $file ) { + // Parallel workers each merge the coverage they collected into a report of their own, + // as they would otherwise merge, and remove, each other's files while those are written. + $coverage_id = self::get_coverage_id(); + $logs_dir = self::$behat_run_dir . '/build/logs'; + $source_dir = '' !== $coverage_id ? "{$logs_dir}/behat-{$coverage_id}" : $logs_dir; + $report = '' !== $coverage_id ? "{$logs_dir}/behat-coverage-{$coverage_id}.xml" : "{$logs_dir}/behat-coverage.xml"; + + if ( ! is_dir( $source_dir ) ) { + return; + } + + foreach ( new DirectoryIterator( $source_dir ) as $file ) { if ( ! $file->isFile() || 'cov' !== $file->getExtension() ) { continue; } @@ -329,7 +340,20 @@ class_exists( Selector::class ) ? ( new Selector() )->forLineCoverage( $filter ) unlink( $file->getPathname() ); } - ( new Clover() )->process( $coverage, self::$behat_run_dir . '/build/logs/behat-coverage.xml' ); + ( new Clover() )->process( $coverage, $report ); + + if ( $source_dir !== $logs_dir ) { + @rmdir( $source_dir ); + } + } + + /** + * Get the identifier of the parallel worker the code coverage is collected for, if any. + * + * @return string + */ + private static function get_coverage_id() { + return (string) preg_replace( '/[^A-Za-z0-9_-]+/', '-', (string) getenv( 'WP_CLI_TEST_COVERAGE_ID' ) ); } /** @@ -506,6 +530,11 @@ private static function get_process_env_variables(): array { $current = getenv( 'WP_CLI_REQUIRE' ); $updated = $current ? "{$current},{$coverage_require_file}" : $coverage_require_file; $env['WP_CLI_REQUIRE'] = $updated; + + $coverage_id = self::get_coverage_id(); + if ( '' !== $coverage_id ) { + $env['WP_CLI_TEST_COVERAGE_ID'] = $coverage_id; + } } $env = array_filter( diff --git a/tests/tests/TestBehatParallel.php b/tests/tests/TestBehatParallel.php new file mode 100644 index 00000000..3d345ddb --- /dev/null +++ b/tests/tests/TestBehatParallel.php @@ -0,0 +1,264 @@ +markTestSkipped( 'The runner does not split the scenarios on Windows.' ); + } + + $this->temp_dir = Utils\get_temp_dir() . uniqid( 'wp-cli-test-behat-parallel-', true ); + mkdir( $this->temp_dir . '/features', 0777, true ); + file_put_contents( $this->temp_dir . '/behat.yml', "default:\n suites:\n default:\n contexts: []\n" ); + + // The runner loads the Gherkin parser and Behat from the project it runs in. + $vendor = dirname( dirname( __DIR__ ) ) . '/vendor'; + if ( ! @symlink( $vendor, $this->temp_dir . '/vendor' ) ) { // phpcs:ignore WordPress.PHP.NoSilencedErrors.Discouraged -- Skipped below. + $this->markTestSkipped( 'The vendor directory cannot be linked into the temporary project.' ); + } + } + + protected function tear_down(): void { + if ( null === $this->temp_dir ) { + parent::tear_down(); + return; + } + + if ( is_link( $this->temp_dir . '/vendor' ) ) { + unlink( $this->temp_dir . '/vendor' ); + } + $this->remove_dir( $this->get_base_dir() ); + $this->remove_dir( $this->temp_dir ); + + parent::tear_down(); + } + + public function test_shards_run_every_scenario_once(): void { + $expected = $this->write_features(); + + $picked = []; + for ( $index = 1; $index <= 3; $index++ ) { + $result = $this->run_shard( "{$index}/3" ); + $this->assertSame( 0, $result['exit_code'], $result['output'] ); + + $shard = $this->get_shard_scenarios(); + $this->assertNotEmpty( $shard, "Shard {$index} of 3 got no scenarios." ); + $this->assertStringContainsString( sprintf( 'Running shard %d of 3, with %d of %d scenarios.', $index, count( $shard ), count( $expected ) ), $result['output'] ); + + $picked = array_merge( $picked, $shard ); + } + + sort( $picked ); + $this->assertSame( $expected, $picked ); + } + + public function test_shards_are_balanced_by_cost(): void { + $this->write_feature( + 'install.feature', + [ + 'Install one' => [ 'Given a WP install' ], + 'Install two' => [ 'Given a WP install' ], + ] + ); + $this->write_feature( + 'cheap.feature', + [ + 'Cheap one' => [ 'Given an empty directory' ], + 'Cheap two' => [ 'Given an empty directory' ], + 'Cheap three' => [ 'Given an empty directory' ], + 'Cheap four' => [ 'Given an empty directory' ], + ] + ); + + // The two installations are the most expensive by far, so each shard gets one of them. + foreach ( [ '1/2', '2/2' ] as $shard ) { + $this->run_shard( $shard ); + $installs = array_filter( + $this->get_shard_scenarios(), + static function ( $path ) { + return false !== strpos( $path, 'install.feature' ); + } + ); + $this->assertCount( 1, $installs, "Shard {$shard}" ); + } + } + + public function test_shard_without_scenarios_succeeds(): void { + $this->write_feature( 'only.feature', [ 'The only one' => [ 'Given an empty directory' ] ] ); + + $result = $this->run_shard( '2/2' ); + + $this->assertSame( 0, $result['exit_code'], $result['output'] ); + $this->assertStringContainsString( 'Running shard 2 of 2, with 0 of 1 scenarios.', $result['output'] ); + $this->assertSame( [], $this->get_shard_scenarios() ); + } + + public function test_invalid_shard_runs_all_scenarios(): void { + $this->write_features(); + + $result = $this->run_shard( '3/2' ); + + $this->assertStringContainsString( 'WP_CLI_TEST_SHARD is not of the form /', $result['output'] ); + $this->assertStringNotContainsString( 'Running shard', $result['output'] ); + } + + public function test_selected_scenarios_ignore_the_shard(): void { + $this->write_features(); + + $result = $this->run_shard( '1/3', [ 'features/a.feature' ] ); + + $this->assertStringNotContainsString( 'Running shard', $result['output'] ); + } + + /** + * Write feature files with a mix of scenarios, including an outline and one using shared resources. + * + * @return string[] Paths of all scenarios, sorted. + */ + private function write_features(): array { + $this->write_feature( + 'a.feature', + [ + 'First' => [ 'Given a WP install', 'When I run `wp option get home`' ], + 'Second' => [ 'Given an empty directory' ], + 'Server' => [ 'Given a WP install', 'And a PHP built-in web server' ], + ] + ); + $this->write_feature( + 'b.feature', + [ + 'Third' => [ 'Given an empty directory', 'When I run `wp cli version`' ], + 'Fourth' => [ 'Given a WP install with Composer' ], + ], + " Scenario Outline: Fifth\n Given an empty directory\n When I run `wp `\n\n Examples:\n | command |\n | cli info |\n | cli version |\n" + ); + $this->write_feature( + 'c.feature', + [ + 'Sixth' => [ 'Given an empty directory' ], + 'Seventh' => [ 'Given a WP install', 'When I run `wp plugin list`' ], + ] + ); + + $paths = []; + foreach ( glob( $this->temp_dir . '/features/*.feature' ) as $file ) { + $lines = file( $file ); + foreach ( $lines as $number => $line ) { + if ( preg_match( '/^\s*Scenario( Outline)?:/', $line ) ) { + $paths[] = 'features/' . basename( $file ) . ':' . ( $number + 1 ); + } + } + } + sort( $paths ); + + return $paths; + } + + /** + * @param string $name File name. + * @param array $scenarios Steps by scenario title. + * @param string $extra Additional Gherkin to append. + */ + private function write_feature( $name, array $scenarios, $extra = '' ): void { + $contents = "Feature: {$name}\n"; + foreach ( $scenarios as $title => $steps ) { + $contents .= "\n Scenario: {$title}\n " . implode( "\n ", $steps ) . "\n"; + } + if ( '' !== $extra ) { + $contents .= "\n" . $extra; + } + + file_put_contents( $this->temp_dir . '/features/' . $name, $contents ); + } + + /** + * Run the runner with a single worker, as run-behat-tests does for a shard. + * + * @param string $shard Value of WP_CLI_TEST_SHARD. + * @param string[] $user_args Arguments passed to run-behat-tests. + * @return array{output: string, exit_code: int} + */ + private function run_shard( $shard, array $user_args = [] ): array { + $script = dirname( dirname( __DIR__ ) ) . '/utils/behat-parallel.php'; + $command = array_merge( [ PHP_BINARY, $script, '1', '--dry-run', '--format=progress', '--' ], $user_args ); + + $env = array_merge( + getenv(), + [ + 'WP_CLI_TEST_SHARD' => $shard, + 'WP_CLI_TEST_DOCKER' => '0', + 'BEHAT_FEATURES_FOLDER' => 'features', + ] + ); + unset( $env['WP_CLI_TEST_COVERAGE'] ); + + $process = proc_open( + implode( ' ', array_map( 'escapeshellarg', $command ) ) . ' 2>&1', + [ 1 => [ 'pipe', 'w' ] ], + $pipes, + $this->temp_dir, + $env + ); + $this->assertIsResource( $process ); + + $output = (string) stream_get_contents( $pipes[1] ); + fclose( $pipes[1] ); + + return [ + 'output' => $output, + 'exit_code' => proc_close( $process ), + ]; + } + + /** + * Get the scenarios the last run handed to Behat for its shard. + * + * @return string[] + */ + private function get_shard_scenarios(): array { + $list = $this->get_base_dir() . '/shard.scenarios'; + $this->assertFileExists( $list ); + + return array_values( array_filter( explode( "\n", (string) file_get_contents( $list ) ) ) ); + } + + /** + * Directory the runner keeps its state in, which depends on the project directory. + */ + private function get_base_dir(): string { + return sys_get_temp_dir() . '/wp-cli-test-parallel-' . substr( md5( (string) realpath( $this->temp_dir ) ), 0, 8 ); + } + + private function remove_dir( string $dir ): void { + if ( ! is_dir( $dir ) ) { + return; + } + $items = new \RecursiveIteratorIterator( + new \RecursiveDirectoryIterator( $dir, \FilesystemIterator::SKIP_DOTS ), + \RecursiveIteratorIterator::CHILD_FIRST + ); + foreach ( $items as $item ) { + $item->isDir() && ! $item->isLink() ? rmdir( (string) $item ) : unlink( (string) $item ); + } + rmdir( $dir ); + } +} diff --git a/utils/behat-parallel.php b/utils/behat-parallel.php index 3de8b7b3..a3be0547 100644 --- a/utils/behat-parallel.php +++ b/utils/behat-parallel.php @@ -157,6 +157,12 @@ function wp_cli_tests_collect_scenarios( $features_folder, $tags ) { $filter = '' !== $tags ? new TagFilter( $tags ) : null; $scenarios = []; + // With code coverage, building a Phar of the bundle itself rewrites its Composer autoloader + // for the duration of the build, which would break the scenarios of the other workers. + $phar_shared = in_array( (string) getenv( 'WP_CLI_TEST_COVERAGE' ), [ 'true', '1' ], true ) + && ! file_exists( 'vendor/wp-cli/wp-cli-bundle/utils/make-phar.php' ) + && file_exists( 'utils/make-phar.php' ); + $files = glob( $features_folder . '/*.feature' ); sort( $files ); @@ -188,7 +194,8 @@ function wp_cli_tests_collect_scenarios( $features_folder, $tags ) { $scenarios[] = [ 'path' => $file . ':' . $scenario->getLine(), 'cost' => $cost, - 'shared' => (bool) preg_match( WP_CLI_TESTS_SHARED_RESOURCES, $text ), + 'shared' => (bool) preg_match( WP_CLI_TESTS_SHARED_RESOURCES, $text ) + || ( $phar_shared && preg_match( '/^a new Phar/m', $text ) ), ]; } } @@ -239,6 +246,64 @@ static function ( $a, $b ) { return $shards; } +/** + * Get the share of the scenarios to run, as set through WP_CLI_TEST_SHARD to "/". + * + * @return array{int, int}|null One-based index and total number of shards, or null to run all of them. + */ +function wp_cli_tests_shard() { + $value = trim( (string) getenv( 'WP_CLI_TEST_SHARD' ) ); + + if ( '' === $value ) { + return null; + } + + if ( ! preg_match( '#^(\d+)/(\d+)$#', $value, $matches ) || (int) $matches[1] < 1 || (int) $matches[1] > (int) $matches[2] ) { + fwrite( STDERR, "Warning: WP_CLI_TEST_SHARD is not of the form /, like 2/3, running all scenarios.\n" ); + return null; + } + + return [ (int) $matches[1], (int) $matches[2] ]; +} + +/** + * Pick one shard of the scenarios, out of a cost-balanced distribution over all shards. + * + * Scenarios that rely on shared resources are spread over the shards as well, as every shard runs + * on a machine of its own. The distribution only depends on the scenarios, so every shard computes + * the same one and together they run every scenario exactly once. + * + * @param array $scenarios + * @param int $index One-based. + * @param int $total + * @return array + */ +function wp_cli_tests_pick_shard( array $scenarios, $index, $total ) { + $order = array_keys( $scenarios ); + + usort( + $order, + static function ( $a, $b ) use ( $scenarios ) { + return ( $scenarios[ $b ]['cost'] - $scenarios[ $a ]['cost'] ) ?: ( $a - $b ); + } + ); + + $load = array_fill( 0, $total, 0 ); + $picked = []; + + foreach ( $order as $key ) { + $shard = (int) array_search( min( $load ), $load, true ); + $load[ $shard ] += $scenarios[ $key ]['cost']; + if ( $shard === $index - 1 ) { + $picked[ $key ] = $scenarios[ $key ]; + } + } + + ksort( $picked ); + + return array_values( $picked ); +} + /** * Get the Docker image of the database server the tests run against on the host, so that the * containers test against the same type and version of database server. @@ -628,7 +693,9 @@ function wp_cli_tests_docker_worker_command( $name, $php_image, $db_image, array } } -if ( ( $workers < 2 && ! $docker ) || DIRECTORY_SEPARATOR === '\\' || ( $selects_scenarios && null === $rerun_paths && ! $docker ) ) { +$shard = $selects_scenarios || null !== $rerun_paths ? null : wp_cli_tests_shard(); + +if ( ( $workers < 2 && ! $docker && null === $shard ) || DIRECTORY_SEPARATOR === '\\' || ( $selects_scenarios && null === $rerun_paths && ! $docker ) ) { wp_cli_tests_run_behat_and_exit( $all_args ); } @@ -737,10 +804,28 @@ static function ( $path ) { $scenarios = wp_cli_tests_collect_scenarios( $features_folder, $tags ); } +if ( null !== $shard ) { + $all_scenarios = count( $scenarios ); + $scenarios = wp_cli_tests_pick_shard( $scenarios, $shard[0], $shard[1] ); + printf( "Running shard %d of %d, with %d of %d scenarios.\n", $shard[0], $shard[1], count( $scenarios ), $all_scenarios ); +} + $workers = min( max( 1, $workers ), max( 1, count( $scenarios ) ) ); if ( $workers < 2 && ! $docker ) { - wp_cli_tests_run_behat_and_exit( $all_args ); + if ( null === $shard ) { + wp_cli_tests_run_behat_and_exit( $all_args ); + } + + if ( ! is_dir( $base_dir ) ) { + mkdir( $base_dir, 0777, true ); + } + $list = "{$base_dir}/shard.scenarios"; + file_put_contents( $list, implode( "\n", array_column( $scenarios, 'path' ) ) . "\n" ); + if ( count( $scenarios ) > 0 ) { + wp_cli_tests_run_behat_and_exit( array_merge( $all_args, [ $list ] ) ); + } + exit( 0 ); } // Containers do not share ports or temporary directories, so any of them can run any scenario. @@ -778,6 +863,9 @@ static function () use ( &$containers ) { $env = getenv(); + // Every worker merges the code coverage of its own scenarios into a report of its own. + $env['WP_CLI_TEST_COVERAGE_ID'] = ( null !== $shard ? "shard-{$shard[0]}-" : '' ) . "worker-{$index}"; + if ( $index > 0 ) { $env['TMPDIR'] = "{$base_dir}/worker-{$index}"; $env['WP_CLI_TEST_DBNAME'] = "{$db_name}_{$index}"; diff --git a/utils/generate-coverage.php b/utils/generate-coverage.php index 007e66ae..e88b14a6 100644 --- a/utils/generate-coverage.php +++ b/utils/generate-coverage.php @@ -96,7 +96,10 @@ static function () use ( $coverage, $feature, $scenario, $step_line, $name, $pro $feature_suffix = preg_replace( '/[^a-z0-9]+/', '-', strtolower( $feature ) ); $scenario_suffix = preg_replace( '/[^a-z0-9]+/', '-', strtolower( $scenario ) ); $db_type = strtolower( getenv( 'WP_CLI_TEST_DBTYPE' ) ); - $destination = "$project_dir/build/logs/$feature_suffix-$scenario_suffix-$step_line-$db_type.cov"; + // Parallel workers each collect into a directory of their own, see FeatureContext::merge_coverage_reports(). + $coverage_id = preg_replace( '/[^A-Za-z0-9_-]+/', '-', (string) getenv( 'WP_CLI_TEST_COVERAGE_ID' ) ); + $coverage_dir = '' !== $coverage_id ? "$project_dir/build/logs/behat-$coverage_id" : "$project_dir/build/logs"; + $destination = "$coverage_dir/$feature_suffix-$scenario_suffix-$step_line-$db_type.cov"; $dir = dirname( $destination ); if ( ! file_exists( $dir ) ) {