From 1634cd990ffe10aea940718cc8d62f2d1a8bcd7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Andr=C3=A9?= Date: Sat, 26 Sep 2026 21:18:24 +0000 Subject: [PATCH 1/5] Compare screenshots through Playwright's own comparator The bridge gains an expectScreenshot command for pages and locators. It calls page._expectScreenshot(), the private method behind @playwright/test's toHaveScreenshot(): screenshots until two consecutive ones match, masks, then pixelmatch or ssim-cie94. The bridge applies the defaults the JS runner adds on its side (animations disabled, caret hidden, CSS scale). Images travel base64 encoded, masks as selectors rebuilt into locators, frames included. Page::expectScreenshot() and Locator::expectScreenshot() expose it as @internal methods returning a ScreenshotComparison. PageInterface and LocatorInterface are left untouched: downstream test doubles implement them. --- bin/lib/handlers.js | 36 ++++++++++ src/Locator/Locator.php | 27 ++++++++ src/Page/Page.php | 19 ++++++ src/Screenshot/ScreenshotComparison.php | 64 +++++++++++++++++ tests/Unit/Locator/LocatorTest.php | 68 +++++++++++++++++++ tests/Unit/Page/PageTest.php | 33 +++++++++ .../Screenshot/ScreenshotComparisonTest.php | 58 ++++++++++++++++ 7 files changed, 305 insertions(+) create mode 100644 src/Screenshot/ScreenshotComparison.php create mode 100644 tests/Unit/Screenshot/ScreenshotComparisonTest.php diff --git a/bin/lib/handlers.js b/bin/lib/handlers.js index 6210335..071449a 100644 --- a/bin/lib/handlers.js +++ b/bin/lib/handlers.js @@ -2,6 +2,40 @@ const { logger, ErrorHandler, CommandRegistry, BaseHandler, PromiseUtils, FrameU const { globalCoordinator } = require('./coordination'); const { PopupCoordinator } = require('./popup-coordinator'); +// Runs Playwright's own toHaveScreenshot machinery: the loop until two +// consecutive screenshots match, masking, and the pixelmatch or ssim-cie94 +// comparison. page._expectScreenshot() is private API, covered by the +// functional suite so a Playwright upgrade that moves it fails there first. +// The defaults below are the ones @playwright/test applies in its runner, +// which the private method does not apply by itself. +async function expectScreenshot(page, locator, command) { + const options = command.options || {}; + const mask = (options.mask || []).map(target => target.frameSelector + ? FrameUtils.resolve(page, target.frameSelector).locator(target.selector) + : page.locator(target.selector)); + + const result = await page._expectScreenshot({ + animations: 'disabled', + caret: 'hide', + scale: 'css', + ...options, + mask, + locator: locator || undefined, + expected: command.expected ? Buffer.from(command.expected, 'base64') : undefined, + isNot: false, + }); + + const encode = buffer => (buffer ? buffer.toString('base64') : null); + + return { + actual: encode(result.actual), + previous: encode(result.previous), + diff: encode(result.diff), + errorMessage: result.errorMessage || null, + timedOut: Boolean(result.timedOut), + }; +} + // The two callbacks below never run in this Node process: Playwright ships // their source to the browser, so their eval() has page scope only, exactly // like the callbacks passed to page.evaluate() elsewhere in this file. @@ -397,6 +431,7 @@ class PageHandler extends BaseHandler { waitForSelector: () => page.waitForSelector(command.selector, command.options), waitForFunction: () => this.waitForFunction(page, command), screenshot: () => PromiseUtils.wrapBinary(page.screenshot(command.options)), + expectScreenshot: () => expectScreenshot(page, null, command), pdf: () => PromiseUtils.wrapBinary(page.pdf(command.options || {})), evaluateHandle: () => this.evaluateHandle(page, command), addScriptTag: () => page.addScriptTag(command.options), @@ -778,6 +813,7 @@ class LocatorHandler extends BaseHandler { getAttribute: () => PromiseUtils.wrapValue(locator.getAttribute(command.name)), selectOption: () => PromiseUtils.wrapValues(locator.selectOption(command.values, command.options)), screenshot: () => PromiseUtils.wrapBinary(locator.screenshot(command.options)), + expectScreenshot: () => expectScreenshot(page, locator, command), evaluate: () => this.evaluateLocator(locator, command), evaluateHandle: () => this.evaluateHandle(locator, command), waitForFunction: () => this.waitForFunction(locator, command), diff --git a/src/Locator/Locator.php b/src/Locator/Locator.php index b9acc17..88b7ce6 100644 --- a/src/Locator/Locator.php +++ b/src/Locator/Locator.php @@ -52,6 +52,7 @@ use Playwright\Locator\Options\WaitForFunctionOptions; use Playwright\Locator\Options\WaitForOptions; use Playwright\Page\PageInterface; +use Playwright\Screenshot\ScreenshotComparison; use Playwright\Transport\TransportInterface; use Psr\Log\LoggerInterface; use Psr\Log\NullLogger; @@ -232,6 +233,32 @@ public function screenshot(?string $path = null, array|LocatorScreenshotOptions return is_string($binary) ? $binary : null; } + /** + * Takes screenshots until two consecutive ones match, then compares the + * last one with $expected. Without $expected, returns the stable screenshot. + * + * @internal Backs toHaveScreenshot(); the shape may change without notice + * + * @param array $options + */ + public function expectScreenshot(?string $expected, array $options = []): ScreenshotComparison + { + $response = $this->sendCommand('locator.expectScreenshot', [ + 'expected' => null === $expected ? null : base64_encode($expected), + 'options' => $options, + ]); + + return ScreenshotComparison::fromResponse($response); + } + + /** + * @internal + */ + public function getFrameSelector(): ?string + { + return $this->frameSelector; + } + /** * @return array */ diff --git a/src/Page/Page.php b/src/Page/Page.php index 3a35201..a01e425 100644 --- a/src/Page/Page.php +++ b/src/Page/Page.php @@ -73,6 +73,7 @@ use Playwright\Regex; use Playwright\Screencast\Screencast; use Playwright\Screencast\ScreencastInterface; +use Playwright\Screenshot\ScreenshotComparison; use Playwright\Screenshot\ScreenshotHelper; use Playwright\Transport\TransportInterface; use Playwright\Video\Video; @@ -396,6 +397,24 @@ public function screenshot(?string $path = null, array|ScreenshotOptions $option return $finalPath; } + /** + * Takes screenshots until two consecutive ones match, then compares the + * last one with $expected. Without $expected, returns the stable screenshot. + * + * @internal Backs toHaveScreenshot(); the shape may change without notice + * + * @param array $options + */ + public function expectScreenshot(?string $expected, array $options = []): ScreenshotComparison + { + $response = $this->sendCommand('expectScreenshot', [ + 'expected' => null === $expected ? null : base64_encode($expected), + 'options' => $options, + ]); + + return ScreenshotComparison::fromResponse($response); + } + /** * Take an auto-generated screenshot with custom suffix. * diff --git a/src/Screenshot/ScreenshotComparison.php b/src/Screenshot/ScreenshotComparison.php new file mode 100644 index 0000000..b1fa4ac --- /dev/null +++ b/src/Screenshot/ScreenshotComparison.php @@ -0,0 +1,64 @@ + $response + */ + public static function fromResponse(array $response): self + { + $binary = static function (mixed $value): ?string { + if (!is_string($value) || '' === $value) { + return null; + } + + $decoded = base64_decode($value, true); + + return false === $decoded ? null : $decoded; + }; + + $errorMessage = $response['errorMessage'] ?? null; + + return new self( + actual: $binary($response['actual'] ?? null), + diff: $binary($response['diff'] ?? null), + previous: $binary($response['previous'] ?? null), + errorMessage: is_string($errorMessage) && '' !== $errorMessage ? $errorMessage : null, + timedOut: true === ($response['timedOut'] ?? false), + ); + } + + public function matches(): bool + { + return null === $this->errorMessage; + } +} diff --git a/tests/Unit/Locator/LocatorTest.php b/tests/Unit/Locator/LocatorTest.php index 7800a32..e8308ac 100644 --- a/tests/Unit/Locator/LocatorTest.php +++ b/tests/Unit/Locator/LocatorTest.php @@ -1213,6 +1213,74 @@ public function testDerivedLocatorsCarryThePageAlong(): void * Rebinds $this->locator to the '.items' selector used by the filter and * combinator tests, which need a different base selector than setUp(). */ + public function testExpectScreenshotSendsTheBaselineAndOptions(): void + { + $this->transport->expects($this->once()) + ->method('send') + ->with([ + 'expected' => base64_encode('baseline'), + 'options' => ['threshold' => 0.1, 'timeout' => 1000], + 'action' => 'locator.expectScreenshot', + 'pageId' => 'page1', + 'selector' => '.element', + ]) + ->willReturn([ + 'actual' => base64_encode('actual'), + 'previous' => null, + 'diff' => base64_encode('diff'), + 'errorMessage' => '12 pixels (ratio 0.01 of all image pixels) are different.', + 'timedOut' => false, + ]); + + $comparison = $this->locator->expectScreenshot('baseline', ['threshold' => 0.1, 'timeout' => 1000]); + + $this->assertSame('actual', $comparison->actual); + $this->assertSame('diff', $comparison->diff); + $this->assertSame('12 pixels (ratio 0.01 of all image pixels) are different.', $comparison->errorMessage); + $this->assertFalse($comparison->matches()); + } + + public function testExpectScreenshotWithoutBaselineAsksForAStableScreenshot(): void + { + $this->transport->expects($this->once()) + ->method('send') + ->with($this->callback(static fn (array $payload): bool => null === $payload['expected'] && [] === $payload['options'])) + ->willReturn(['actual' => base64_encode('stable'), 'errorMessage' => null, 'timedOut' => false]); + + $comparison = $this->locator->expectScreenshot(null); + + $this->assertSame('stable', $comparison->actual); + $this->assertTrue($comparison->matches()); + } + + public function testExpectScreenshotInAFrameSendsTheFrameSelector(): void + { + $locator = new Locator($this->transport, 'page1', '.inner', 'iframe#preview'); + + $this->transport->expects($this->once()) + ->method('send') + ->with($this->callback(static fn (array $payload): bool => 'iframe#preview' === $payload['frameSelector'] && '.inner' === $payload['selector'])) + ->willReturn(['errorMessage' => null]); + + $this->assertTrue($locator->expectScreenshot('baseline')->matches()); + } + + public function testExpectScreenshotSurfacesBridgeErrors(): void + { + $this->transport->method('send')->willReturn(['error' => 'page._expectScreenshot is not a function']); + + $this->expectException(PlaywrightException::class); + $this->expectExceptionMessage('page._expectScreenshot is not a function'); + + $this->locator->expectScreenshot('baseline'); + } + + public function testGetFrameSelector(): void + { + $this->assertNull($this->locator->getFrameSelector()); + $this->assertSame('iframe#preview', (new Locator($this->transport, 'page1', '.inner', 'iframe#preview'))->getFrameSelector()); + } + private function useItemsLocator(): void { $this->locator = new Locator($this->transport, 'page1', '.items'); diff --git a/tests/Unit/Page/PageTest.php b/tests/Unit/Page/PageTest.php index 4747e77..fe8f3cb 100644 --- a/tests/Unit/Page/PageTest.php +++ b/tests/Unit/Page/PageTest.php @@ -1235,6 +1235,39 @@ public function testQueriedFramesRetainTheirOriginatingPage(): void $this->assertSame($this->page, $frame->page()); } + public function testExpectScreenshotSendsAPageCommand(): void + { + $this->transport->expects($this->once()) + ->method('send') + ->with([ + 'expected' => base64_encode('baseline'), + 'options' => ['fullPage' => true, 'timeout' => 1000], + 'action' => 'page.expectScreenshot', + 'pageId' => 'page-id', + ]) + ->willReturn(['actual' => null, 'diff' => null, 'errorMessage' => null, 'timedOut' => false]); + + $this->assertTrue($this->page->expectScreenshot('baseline', ['fullPage' => true, 'timeout' => 1000])->matches()); + } + + public function testExpectScreenshotWithoutBaselineReturnsTheStableScreenshot(): void + { + $this->transport->expects($this->once()) + ->method('send') + ->with($this->callback(static fn (array $payload): bool => null === $payload['expected'])) + ->willReturn(['actual' => base64_encode('stable'), 'errorMessage' => null, 'timedOut' => false]); + + $this->assertSame('stable', $this->page->expectScreenshot(null)->actual); + } + + public function testFrameLocatorLocatorsKeepTheirFrameForMasks(): void + { + $locator = $this->page->frameLocator('iframe#preview')->locator('.date'); + + $this->assertInstanceOf(Locator::class, $locator); + $this->assertSame('iframe#preview', $locator->getFrameSelector()); + } + private function createPage(string $pageId = 'page-1'): Page { return new Page($this->transport, $this->context, $pageId, new PlaywrightConfig()); diff --git a/tests/Unit/Screenshot/ScreenshotComparisonTest.php b/tests/Unit/Screenshot/ScreenshotComparisonTest.php new file mode 100644 index 0000000..828b7dc --- /dev/null +++ b/tests/Unit/Screenshot/ScreenshotComparisonTest.php @@ -0,0 +1,58 @@ + base64_encode('actual'), + 'previous' => base64_encode('previous'), + 'diff' => base64_encode('diff'), + 'errorMessage' => '3 pixels are different.', + 'timedOut' => false, + ]); + + $this->assertSame('actual', $comparison->actual); + $this->assertSame('previous', $comparison->previous); + $this->assertSame('diff', $comparison->diff); + $this->assertSame('3 pixels are different.', $comparison->errorMessage); + $this->assertFalse($comparison->timedOut); + $this->assertFalse($comparison->matches()); + } + + public function testMatchesWhenTheBridgeReportsNoError(): void + { + $comparison = ScreenshotComparison::fromResponse(['actual' => null, 'diff' => null, 'errorMessage' => null, 'timedOut' => false]); + + $this->assertTrue($comparison->matches()); + $this->assertNull($comparison->actual); + } + + public function testIgnoresMalformedValues(): void + { + $comparison = ScreenshotComparison::fromResponse(['actual' => '***', 'errorMessage' => '', 'timedOut' => 'yes']); + + $this->assertNull($comparison->actual); + $this->assertNull($comparison->errorMessage); + $this->assertFalse($comparison->timedOut); + } +} From e175ba8a7013778f9c07e642012e674cd78ad6b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Andr=C3=A9?= Date: Sat, 26 Sep 2026 21:18:24 +0000 Subject: [PATCH 2/5] Add the toHaveScreenshot visual regression assertion toHaveScreenshot() on page and locator assertions, and on expect() in test cases, compares a stable screenshot with a baseline image, as in @playwright/test: - Baselines live next to the test file, .php-snapshots/, named after the browser of the page under test and the platform: card-chromium-linux.png. PNG by default, lossless WebP with .webp. - PLAYWRIGHT_UPDATE_SNAPSHOTS takes the --update-snapshots values: none, missing (default), changed, all. "all" records without comparing. - A failure writes card-expected, card-actual and card-diff to test-failures/--/, removed once the baseline matches or is recorded again. - ToHaveScreenshotOptions carries the @playwright/test defaults and rejects invalid values on construction. --- CHANGELOG.md | 3 + .../Internal/AbstractAssertions.php | 27 ++ .../Internal/ScreenshotExpectation.php | 226 +++++++++ src/Assertions/LocatorAssertions.php | 38 ++ src/Assertions/LocatorAssertionsInterface.php | 17 + .../Options/ToHaveScreenshotOptions.php | 189 ++++++++ src/Assertions/PageAssertions.php | 34 ++ src/Assertions/PageAssertionsInterface.php | 17 + src/Testing/Expect.php | 69 ++- src/Testing/ExpectDecorator.php | 8 + src/Testing/ExpectInterface.php | 13 + src/Testing/PlaywrightTestCaseTrait.php | 26 +- .../Screenshot/ToHaveScreenshotTest.php | 339 ++++++++++++++ .../Internal/ScreenshotExpectationTest.php | 434 ++++++++++++++++++ .../Unit/Assertions/LocatorAssertionsTest.php | 127 +++++ .../Options/ToHaveScreenshotOptionsTest.php | 145 ++++++ tests/Unit/Assertions/PageAssertionsTest.php | 89 ++++ tests/Unit/Testing/ExpectFactoryTest.php | 277 +++++++++++ 18 files changed, 2073 insertions(+), 5 deletions(-) create mode 100644 src/Assertions/Internal/ScreenshotExpectation.php create mode 100644 src/Assertions/Options/ToHaveScreenshotOptions.php create mode 100644 tests/Functional/Screenshot/ToHaveScreenshotTest.php create mode 100644 tests/Unit/Assertions/Internal/ScreenshotExpectationTest.php create mode 100644 tests/Unit/Assertions/Options/ToHaveScreenshotOptionsTest.php diff --git a/CHANGELOG.md b/CHANGELOG.md index 5546de8..db0dfa6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ ## [Unreleased] +### Added +- `toHaveScreenshot()` visual regression assertion on pages and locators, backed by Playwright's own screenshot comparison, with PNG or lossless WebP baselines + ## [1.5.0] - 2026-09-20 ### Added diff --git a/src/Assertions/Internal/AbstractAssertions.php b/src/Assertions/Internal/AbstractAssertions.php index 87ff4a8..a56fde3 100644 --- a/src/Assertions/Internal/AbstractAssertions.php +++ b/src/Assertions/Internal/AbstractAssertions.php @@ -72,6 +72,33 @@ protected function assertCondition( } } + /** + * Runs a matcher that brings its own waiting, such as a screenshot + * comparison, inside the same tracing group as polled matchers. + * + * @param callable(int): void $assertion receives the timeout in milliseconds + */ + protected function assertWithoutPolling(string $matcher, ?int $timeoutMs, callable $assertion): void + { + if ($this->negated) { + $this->negated = false; + + throw new \LogicException(sprintf('%s() cannot be negated.', $matcher)); + } + + if (null !== $this->tracing) { + $this->tracing->group(sprintf('expect(%s).%s', $this->subjectName(), $matcher)); + } + + try { + $assertion($timeoutMs ?? $this->timeoutMs); + } finally { + if (null !== $this->tracing) { + $this->tracing->groupEnd(); + } + } + } + abstract protected function subjectName(): string; /** diff --git a/src/Assertions/Internal/ScreenshotExpectation.php b/src/Assertions/Internal/ScreenshotExpectation.php new file mode 100644 index 0000000..7bedb80 --- /dev/null +++ b/src/Assertions/Internal/ScreenshotExpectation.php @@ -0,0 +1,226 @@ +): ScreenshotComparison $compare + */ + public function __construct( + private readonly \Closure $compare, + private readonly string $subject, + private readonly ?string $updateMode = null, + private readonly ?string $outputDir = null, + private readonly ?string $variant = null, + ) { + } + + public function assert(string $path, ToHaveScreenshotOptions $options, int $timeoutMs): void + { + $mode = $this->updateMode(); + $payload = $options->toArray() + ['timeout' => $options->timeoutMs ?? $timeoutMs, 'type' => $this->imageType($path)]; + + $expected = is_file($path) ? @file_get_contents($path) : null; + if (false === $expected) { + throw new \RuntimeException(sprintf('Unable to read the snapshot "%s".', $path)); + } + + // "all" records every screenshot, like --update-snapshots=all: no + // comparison, Playwright returns no image when a comparison matches. + if (null === $expected || 'all' === $mode) { + $this->record($path, $mode, null === $expected, ($this->compare)(null, $payload), $options); + + return; + } + + $result = ($this->compare)($expected, $payload); + + if ($result->matches()) { + $this->removeArtifacts($path); + + return; + } + + if ('changed' === $mode && null !== $result->actual && !$result->timedOut) { + $this->write($path, $result->actual); + $this->removeArtifacts($path); + + return; + } + + $artifacts = $this->writeArtifacts($path, $expected, $result); + + throw new AssertionException($this->failureMessage($options, $result, $path, $artifacts), actual: $artifacts['actual'] ?? null, expected: $path); + } + + private function record(string $path, string $mode, bool $missing, ScreenshotComparison $result, ToHaveScreenshotOptions $options): void + { + if (null === $result->actual || !$result->matches()) { + throw new AssertionException($options->message ?? sprintf('Failed to take a stable screenshot of the %s: %s', $this->subject, $result->errorMessage ?? 'no screenshot returned')); + } + + if ($missing && 'none' === $mode) { + throw new AssertionException($options->message ?? sprintf('A snapshot doesn\'t exist at %s. Run with %s=missing to write it.', $path, self::UPDATE_ENV)); + } + + $this->write($path, $result->actual); + $this->removeArtifacts($path); + + if ($missing && 'missing' === $mode) { + throw new AssertionException($options->message ?? sprintf('A snapshot doesn\'t exist at %s, writing actual.', $path)); + } + } + + /** + * Images of an earlier failure would read as a current one: they go as + * soon as the baseline matches or is recorded again. + */ + private function removeArtifacts(string $path): void + { + $stem = $this->artifactStem($path); + $extension = '.'.$this->imageType($path); + + foreach ([$stem.'-expected'.$extension, $stem.'-actual'.$extension, $stem.'-diff.png'] as $file) { + if (is_file($file)) { + unlink($file); + } + } + + if (is_dir($this->outputDir()) && [] === array_diff(scandir($this->outputDir()) ?: [], ['.', '..'])) { + rmdir($this->outputDir()); + } + } + + /** + * @return array<'expected'|'actual'|'diff', string> + */ + private function writeArtifacts(string $path, string $expected, ScreenshotComparison $result): array + { + $stem = $this->artifactStem($path); + $extension = '.'.$this->imageType($path); + + $artifacts = ['expected' => $stem.'-expected'.$extension]; + $this->write($artifacts['expected'], $expected); + + if (null !== $result->actual) { + $artifacts['actual'] = $stem.'-actual'.$extension; + $this->write($artifacts['actual'], $result->actual); + } + if (null !== $result->diff) { + $artifacts['diff'] = $stem.'-diff.png'; + $this->write($artifacts['diff'], $result->diff); + } + + return $artifacts; + } + + /** + * @param array<'expected'|'actual'|'diff', string> $artifacts + */ + private function failureMessage(ToHaveScreenshotOptions $options, ScreenshotComparison $result, string $path, array $artifacts): string + { + $message = $options->message ?? sprintf('Screenshot of the %s does not match %s.', $this->subject, $path); + + $lines = [$message, (string) $result->errorMessage]; + foreach ($artifacts as $kind => $file) { + $lines[] = sprintf(' %-9s %s', $kind.':', $file); + } + $lines[] = sprintf('Run with %s=changed to accept the new screenshot.', self::UPDATE_ENV); + + return implode("\n", $lines); + } + + /** + * The baseline extension picks the image format. WebP is lossless here, + * smaller and faster to compare; JPEG is rejected, its compression noise + * would fail every comparison. + * + * @return 'png'|'webp' + */ + private function imageType(string $path): string + { + return match (strtolower(pathinfo($path, \PATHINFO_EXTENSION))) { + 'png' => 'png', + 'webp' => 'webp', + default => throw new \InvalidArgumentException(sprintf('Snapshot "%s" must be a .png or .webp file.', $path)), + }; + } + + private function updateMode(): string + { + $mode = $this->updateMode ?? $_SERVER[self::UPDATE_ENV] ?? getenv(self::UPDATE_ENV); + if (!is_string($mode) || '' === $mode) { + return 'missing'; + } + + if (!in_array($mode, self::UPDATE_MODES, true)) { + throw new \InvalidArgumentException(sprintf('%s must be one of "%s", "%s" given.', self::UPDATE_ENV, implode('", "', self::UPDATE_MODES), $mode)); + } + + return $mode; + } + + /** + * Failure images drop the variant of the baseline name, which their + * directory already carries: "card-chromium-linux.png" fails as + * "card-actual.png", as in Playwright JS. + */ + private function artifactStem(string $path): string + { + $stem = pathinfo($path, \PATHINFO_FILENAME); + if (null !== $this->variant && '' !== $this->variant && str_ends_with($stem, '-'.$this->variant)) { + $stem = substr($stem, 0, -\strlen($this->variant) - 1); + } + + return $this->outputDir().'/'.$stem; + } + + private function outputDir(): string + { + return $this->outputDir ?? getcwd().'/test-failures/snapshots'; + } + + private function write(string $file, string $contents): void + { + $dir = dirname($file); + if (!is_dir($dir) && !@mkdir($dir, 0777, true) && !is_dir($dir)) { + throw new \RuntimeException(sprintf('Directory "%s" was not created.', $dir)); + } + + if (false === @file_put_contents($file, $contents)) { + throw new \RuntimeException(sprintf('Unable to write "%s".', $file)); + } + } +} diff --git a/src/Assertions/LocatorAssertions.php b/src/Assertions/LocatorAssertions.php index 28710a2..f7576d9 100644 --- a/src/Assertions/LocatorAssertions.php +++ b/src/Assertions/LocatorAssertions.php @@ -16,6 +16,9 @@ use Playwright\Assertions\Internal\AbstractAssertions; use Playwright\Assertions\Internal\AriaSnapshot; +use Playwright\Assertions\Internal\ScreenshotExpectation; +use Playwright\Assertions\Options\ToHaveScreenshotOptions; +use Playwright\Locator\Locator; use Playwright\Locator\LocatorInterface; use Playwright\Tracing\TracingInterface; @@ -83,9 +86,17 @@ final class LocatorAssertions extends AbstractAssertions implements LocatorAsser } JS; + /** + * @param string|null $screenshotOutputDir Where toHaveScreenshot() writes the expected, actual and diff images of a + * failure. Defaults to test-failures/snapshots/ in the working directory. + * @param string|null $screenshotVariant Suffix of the baseline names, such as "chromium-linux", left out of the + * failure image names: "card-chromium-linux.png" fails as "card-actual.png". + */ public function __construct( private readonly LocatorInterface $locator, ?TracingInterface $tracing = null, + private readonly ?string $screenshotOutputDir = null, + private readonly ?string $screenshotVariant = null, ) { parent::__construct($tracing); } @@ -630,6 +641,33 @@ private static function classTokens(string $classAttribute): array return false === $tokens ? [] : $tokens; } + public function toHaveScreenshot(string $path, ?ToHaveScreenshotOptions $options = null): self + { + $options ??= new ToHaveScreenshotOptions(); + $subject = $this->locator; + if (!$subject instanceof Locator) { + throw new \LogicException(sprintf('toHaveScreenshot() needs a %s created by Playwright PHP, %s given.', Locator::class, get_debug_type($subject))); + } + + if ($options->fullPage || null !== $options->clip) { + throw new \InvalidArgumentException('The fullPage and clip options apply to a page screenshot, an element is captured whole.'); + } + + $outputDir = $this->screenshotOutputDir; + $variant = $this->screenshotVariant; + $this->assertWithoutPolling('toHaveScreenshot', $options->timeoutMs, static function (int $timeoutMs) use ($subject, $path, $options, $outputDir, $variant): void { + $expectation = new ScreenshotExpectation( + $subject->expectScreenshot(...), + 'locator', + outputDir: $outputDir, + variant: $variant, + ); + $expectation->assert($path, $options, $timeoutMs); + }); + + return $this; + } + protected function subjectName(): string { return $this->locator->getSelector(); diff --git a/src/Assertions/LocatorAssertionsInterface.php b/src/Assertions/LocatorAssertionsInterface.php index 69de067..3354b1c 100644 --- a/src/Assertions/LocatorAssertionsInterface.php +++ b/src/Assertions/LocatorAssertionsInterface.php @@ -14,6 +14,8 @@ namespace Playwright\Assertions; +use Playwright\Assertions\Options\ToHaveScreenshotOptions; + interface LocatorAssertionsInterface { /** @@ -147,6 +149,21 @@ public function toHaveAccessibleErrorMessage(string $expected, ?AssertionOptions */ public function toMatchAriaSnapshot(string $expected, ?AssertionOptions $options = null): self; + /** + * Asserts a stable screenshot of the element matches the baseline image at + * $path, a .png or a lossless .webp. + * + * Screenshots are taken until two consecutive ones are identical, then + * compared with the baseline by Playwright's own comparator. A missing + * baseline is written and the assertion fails; PLAYWRIGHT_UPDATE_SNAPSHOTS + * ("none", "missing", "changed", "all") controls what gets rewritten. On + * failure, the expected, actual and diff images are written to the + * screenshot output directory, test-failures/snapshots/ by default, and + * removed once the baseline matches or is recorded again. Negation throws + * a LogicException. + */ + public function toHaveScreenshot(string $path, ?ToHaveScreenshotOptions $options = null): self; + public function not(): self; public function withTimeout(int $timeoutMs): self; diff --git a/src/Assertions/Options/ToHaveScreenshotOptions.php b/src/Assertions/Options/ToHaveScreenshotOptions.php new file mode 100644 index 0000000..75c8557 --- /dev/null +++ b/src/Assertions/Options/ToHaveScreenshotOptions.php @@ -0,0 +1,189 @@ +locator('.date')]) + * + * Invalid values throw an \InvalidArgumentException on construction, so a + * typo fails in PHP before reaching the browser. + */ +final readonly class ToHaveScreenshotOptions +{ + public const COMPARATOR_PIXELMATCH = 'pixelmatch'; + public const COMPARATOR_SSIM_CIE94 = 'ssim-cie94'; + + private const COMPARATORS = [self::COMPARATOR_PIXELMATCH, self::COMPARATOR_SSIM_CIE94]; + private const ANIMATIONS = ['disabled', 'allow']; + private const CARETS = ['hide', 'initial']; + private const SCALES = ['css', 'device']; + + /** + * @param float $threshold Color distance tolerated for each pixel, from 0 (exact) to 1 (anything). + * Used by pixelmatch only. 0.2 forgives anti-aliasing, and also close + * shades of one hue: lower it to 0.1 to catch a design token change. + * @param int|null $maxDiffPixels Number of differing pixels tolerated. None by default: any differing + * pixel fails. When set with $maxDiffPixelRatio, the stricter one wins. + * @param float|null $maxDiffPixelRatio Share of differing pixels tolerated, from 0 to 1. None by default. + * @param 'pixelmatch'|'ssim-cie94' $comparator pixelmatch compares pixel colors. ssim-cie94 compares local structure + * and perceived color: steadier across rendering noise, stricter on + * color shifts, slower, experimental in Playwright. + * @param list $mask Elements covered with a solid box of $maskColor before the comparison, + * for dates, avatars or ads. A locator obtained through frameLocator() + * masks inside its iframe. + * @param string $maskColor Any CSS color. Magenta by default, a color no interface uses. + * @param string|null $style CSS applied while the screenshot is taken, to hide or freeze dynamic + * parts. It pierces the Shadow DOM and applies to iframes. + * @param bool $fullPage Captures the whole scrollable page. Page assertions only. + * @param array{x: float, y: float, width: float, height: float}|null $clip region of the page to capture, in CSS pixels + * @param bool $omitBackground transparent background instead of the default white one + * @param 'disabled'|'allow' $animations "disabled" fast-forwards finite CSS animations and transitions to their + * end and cancels infinite ones, so the page can settle + * @param 'hide'|'initial' $caret "hide" hides the text caret, which blinks + * @param 'css'|'device' $scale "css" takes one pixel per CSS pixel, so baselines match across screen + * densities. "device" is sharper and larger on high-DPI screens. + * @param int|null $timeoutMs Time allowed to get two identical consecutive screenshots, in + * milliseconds. Defaults to the assertion timeout. + * @param string|null $message replaces the first line of the failure message + */ + public function __construct( + public float $threshold = 0.2, + public ?int $maxDiffPixels = null, + public ?float $maxDiffPixelRatio = null, + public string $comparator = self::COMPARATOR_PIXELMATCH, + public array $mask = [], + public string $maskColor = '#FF00FF', + public ?string $style = null, + public bool $fullPage = false, + public ?array $clip = null, + public bool $omitBackground = false, + public string $animations = 'disabled', + public string $caret = 'hide', + public string $scale = 'css', + public ?int $timeoutMs = null, + public ?string $message = null, + ) { + self::assertRange('threshold', $threshold); + if (null !== $maxDiffPixelRatio) { + self::assertRange('maxDiffPixelRatio', $maxDiffPixelRatio); + } + if (null !== $maxDiffPixels && $maxDiffPixels < 0) { + throw new \InvalidArgumentException(sprintf('The maxDiffPixels option must be 0 or more, %d given.', $maxDiffPixels)); + } + if (null !== $timeoutMs && $timeoutMs < 0) { + throw new \InvalidArgumentException(sprintf('The timeoutMs option must be 0 or more, %d given.', $timeoutMs)); + } + + self::assertChoice('comparator', $comparator, self::COMPARATORS); + self::assertChoice('animations', $animations, self::ANIMATIONS); + self::assertChoice('caret', $caret, self::CARETS); + self::assertChoice('scale', $scale, self::SCALES); + + if ('' === trim($maskColor)) { + throw new \InvalidArgumentException('The maskColor option must be a CSS color, an empty string given.'); + } + + foreach ($mask as $index => $locator) { + if (!$locator instanceof LocatorInterface) { + throw new \InvalidArgumentException(sprintf('The mask option expects %s instances, %s given at index %s.', LocatorInterface::class, get_debug_type($locator), $index)); + } + } + + if (null !== $clip) { + self::assertClip($clip); + } + } + + /** + * Options sent to the bridge. The timeout and the message are applied on + * the PHP side and are not part of it. + * + * @return array + */ + public function toArray(): array + { + $options = array_filter([ + 'threshold' => $this->threshold, + 'maxDiffPixels' => $this->maxDiffPixels, + 'maxDiffPixelRatio' => $this->maxDiffPixelRatio, + 'comparator' => $this->comparator, + 'maskColor' => $this->maskColor, + 'style' => $this->style, + 'fullPage' => $this->fullPage, + 'clip' => $this->clip, + 'omitBackground' => $this->omitBackground, + 'animations' => $this->animations, + 'caret' => $this->caret, + 'scale' => $this->scale, + ], static fn (mixed $value): bool => null !== $value); + + if ([] !== $this->mask) { + $options['mask'] = array_map( + static fn (LocatorInterface $locator): array => array_filter([ + 'selector' => $locator->getSelector(), + 'frameSelector' => $locator instanceof Locator ? $locator->getFrameSelector() : null, + ], static fn (?string $value): bool => null !== $value), + $this->mask, + ); + } + + return $options; + } + + private static function assertRange(string $option, float $value): void + { + if ($value < 0 || $value > 1) { + throw new \InvalidArgumentException(sprintf('The %s option must be between 0 and 1, %s given.', $option, $value)); + } + } + + /** + * @param list $choices + */ + private static function assertChoice(string $option, string $value, array $choices): void + { + if (!in_array($value, $choices, true)) { + throw new \InvalidArgumentException(sprintf('The %s option must be one of "%s", "%s" given.', $option, implode('", "', $choices), $value)); + } + } + + /** + * @param array $clip + */ + private static function assertClip(array $clip): void + { + foreach (['x', 'y', 'width', 'height'] as $key) { + if (!isset($clip[$key]) || !is_int($clip[$key]) && !is_float($clip[$key])) { + throw new \InvalidArgumentException(sprintf('The clip option needs numeric "x", "y", "width" and "height" keys, "%s" is missing or not a number.', $key)); + } + } + + if ($clip['width'] <= 0 || $clip['height'] <= 0) { + throw new \InvalidArgumentException('The clip option needs a width and a height greater than 0.'); + } + + if ([] !== array_diff(array_keys($clip), ['x', 'y', 'width', 'height'])) { + throw new \InvalidArgumentException(sprintf('The clip option only accepts "x", "y", "width" and "height" keys, "%s" given.', implode('", "', array_keys($clip)))); + } + } +} diff --git a/src/Assertions/PageAssertions.php b/src/Assertions/PageAssertions.php index a4ce047..eadfa27 100644 --- a/src/Assertions/PageAssertions.php +++ b/src/Assertions/PageAssertions.php @@ -16,14 +16,25 @@ use Playwright\Assertions\Internal\AbstractAssertions; use Playwright\Assertions\Internal\AriaSnapshot; +use Playwright\Assertions\Internal\ScreenshotExpectation; +use Playwright\Assertions\Options\ToHaveScreenshotOptions; +use Playwright\Page\Page; use Playwright\Page\PageInterface; use Playwright\Tracing\TracingInterface; final class PageAssertions extends AbstractAssertions implements PageAssertionsInterface { + /** + * @param string|null $screenshotOutputDir Where toHaveScreenshot() writes the expected, actual and diff images of a + * failure. Defaults to test-failures/snapshots/ in the working directory. + * @param string|null $screenshotVariant Suffix of the baseline names, such as "chromium-linux", left out of the + * failure image names: "card-chromium-linux.png" fails as "card-actual.png". + */ public function __construct( private readonly PageInterface $page, ?TracingInterface $tracing = null, + private readonly ?string $screenshotOutputDir = null, + private readonly ?string $screenshotVariant = null, ) { parent::__construct($tracing); } @@ -96,6 +107,29 @@ public function toMatchAriaSnapshot(string $expected, ?AssertionOptions $options return $this; } + public function toHaveScreenshot(string $path, ?ToHaveScreenshotOptions $options = null): self + { + $options ??= new ToHaveScreenshotOptions(); + $subject = $this->page; + if (!$subject instanceof Page) { + throw new \LogicException(sprintf('toHaveScreenshot() needs a %s created by Playwright PHP, %s given.', Page::class, get_debug_type($subject))); + } + + $outputDir = $this->screenshotOutputDir; + $variant = $this->screenshotVariant; + $this->assertWithoutPolling('toHaveScreenshot', $options->timeoutMs, static function (int $timeoutMs) use ($subject, $path, $options, $outputDir, $variant): void { + $expectation = new ScreenshotExpectation( + $subject->expectScreenshot(...), + 'page', + outputDir: $outputDir, + variant: $variant, + ); + $expectation->assert($path, $options, $timeoutMs); + }); + + return $this; + } + protected function subjectName(): string { return 'page'; diff --git a/src/Assertions/PageAssertionsInterface.php b/src/Assertions/PageAssertionsInterface.php index 5f534cb..24c531f 100644 --- a/src/Assertions/PageAssertionsInterface.php +++ b/src/Assertions/PageAssertionsInterface.php @@ -14,6 +14,8 @@ namespace Playwright\Assertions; +use Playwright\Assertions\Options\ToHaveScreenshotOptions; + interface PageAssertionsInterface { /** @return $this */ @@ -33,6 +35,21 @@ public function toHaveURL(string|\Stringable $expected, ?AssertionOptions $optio */ public function toMatchAriaSnapshot(string $expected, ?AssertionOptions $options = null): self; + /** + * Asserts a stable screenshot of the page matches the baseline image at + * $path, a .png or a lossless .webp. + * + * Screenshots are taken until two consecutive ones are identical, then + * compared with the baseline by Playwright's own comparator. A missing + * baseline is written and the assertion fails; PLAYWRIGHT_UPDATE_SNAPSHOTS + * ("none", "missing", "changed", "all") controls what gets rewritten. On + * failure, the expected, actual and diff images are written to the + * screenshot output directory, test-failures/snapshots/ by default, and + * removed once the baseline matches or is recorded again. Negation throws + * a LogicException. + */ + public function toHaveScreenshot(string $path, ?ToHaveScreenshotOptions $options = null): self; + /** @return $this */ public function not(): self; diff --git a/src/Testing/Expect.php b/src/Testing/Expect.php index 8e2279e..f529fe9 100644 --- a/src/Testing/Expect.php +++ b/src/Testing/Expect.php @@ -16,8 +16,10 @@ use Playwright\Assertions\LocatorAssertions; use Playwright\Assertions\LocatorAssertionsInterface; +use Playwright\Assertions\Options\ToHaveScreenshotOptions; use Playwright\Assertions\PageAssertions; use Playwright\Assertions\PageAssertionsInterface; +use Playwright\Exception\RuntimeException; use Playwright\Locator\LocatorInterface; use Playwright\Page\PageInterface; use Playwright\Tracing\TracingInterface; @@ -26,11 +28,32 @@ final class Expect implements ExpectInterface { private readonly LocatorAssertionsInterface|PageAssertionsInterface $assertions; - public function __construct(LocatorInterface|PageInterface $subject, ?TracingInterface $tracing = null) - { + /** + * Browser and platform of the run, "chromium-linux": the suffix of relative + * baseline names, as {projectName}-{platform} in @playwright/test. + */ + private readonly string $variant; + + /** + * @param string|null $outputDir Directory of the test for failure images. The browser is appended, as Playwright JS + * appends the project: each browser of a test gets its own directory. + */ + public function __construct( + LocatorInterface|PageInterface $subject, + ?TracingInterface $tracing = null, + private readonly ?string $snapshotDir = null, + ?string $outputDir = null, + ) { + $browser = self::browserName($subject); + $this->variant = implode('-', array_filter([$browser, self::platform()])); + + if (null !== $outputDir && '' !== $browser) { + $outputDir .= '-'.$browser; + } + $this->assertions = $subject instanceof LocatorInterface - ? new LocatorAssertions($subject, $tracing) - : new PageAssertions($subject, $tracing); + ? new LocatorAssertions($subject, $tracing, $outputDir, $this->variant) + : new PageAssertions($subject, $tracing, $outputDir, $this->variant); $this->assertions->withPollInterval(100); } @@ -158,6 +181,44 @@ public function toHaveURL(string $url): void $this->page(__FUNCTION__)->toHaveURL($url); } + public function toHaveScreenshot(string $name, ?ToHaveScreenshotOptions $options = null): void + { + $this->assertions->toHaveScreenshot($this->snapshotPath($name), $options); + } + + private function snapshotPath(string $name): string + { + if (str_starts_with($name, '/') || 1 === preg_match('~^[a-zA-Z]:[\\\\/]~', $name)) { + return $name; + } + + $extension = pathinfo($name, \PATHINFO_EXTENSION); + $stem = '' === $extension ? $name : substr($name, 0, -\strlen($extension) - 1); + + return sprintf('%s/%s-%s.%s', $this->snapshotDir ?? getcwd(), $stem, $this->variant, '' === $extension ? 'png' : $extension); + } + + /** + * The browser rendering the subject, read from the page under test. + * Empty when it cannot be known: a persistent context has no browser + * object, a locator built by hand has no page. + */ + private static function browserName(LocatorInterface|PageInterface $subject): string + { + try { + $page = $subject instanceof LocatorInterface ? $subject->page() : $subject; + + return $page->context()->browser()?->browserType()->value ?? ''; + } catch (RuntimeException) { + return ''; + } + } + + private static function platform(): string + { + return 'Windows' === \PHP_OS_FAMILY ? 'win32' : strtolower(\PHP_OS_FAMILY); + } + private function locator(string $matcher): LocatorAssertionsInterface { if (!$this->assertions instanceof LocatorAssertionsInterface) { diff --git a/src/Testing/ExpectDecorator.php b/src/Testing/ExpectDecorator.php index 5230ed3..7c8f3ed 100644 --- a/src/Testing/ExpectDecorator.php +++ b/src/Testing/ExpectDecorator.php @@ -14,6 +14,8 @@ namespace Playwright\Testing; +use Playwright\Assertions\Options\ToHaveScreenshotOptions; + /** * Decorator for Expect that automatically increments PHPUnit assertion count when available. * This allows the Expect class to work independently (without PHPUnit) while still @@ -152,6 +154,12 @@ public function toHaveURL(string $url): void $this->recordAssertion(1); } + public function toHaveScreenshot(string $name, ?ToHaveScreenshotOptions $options = null): void + { + $this->expect->toHaveScreenshot($name, $options); + $this->recordAssertion(1); + } + public function not(): self { $this->expect->not(); diff --git a/src/Testing/ExpectInterface.php b/src/Testing/ExpectInterface.php index 98bcdff..50aed7b 100644 --- a/src/Testing/ExpectInterface.php +++ b/src/Testing/ExpectInterface.php @@ -14,6 +14,8 @@ namespace Playwright\Testing; +use Playwright\Assertions\Options\ToHaveScreenshotOptions; + interface ExpectInterface { public function toBeAttached(): void; @@ -65,6 +67,17 @@ public function toHaveTitle(string $title): void; public function toHaveURL(string $url): void; + /** + * Compares a stable screenshot with a baseline image. + * + * A relative $name resolves in the snapshot directory of the test, suffixed + * with the browser and the platform read from the page under test: + * "card.png" in tests/CardTest.php, run in Chromium on Linux, is stored as + * tests/CardTest.php-snapshots/card-chromium-linux.png. Without extension, + * ".png" is added. Name it ".webp" for a lossless WebP baseline. + */ + public function toHaveScreenshot(string $name, ?ToHaveScreenshotOptions $options = null): void; + public function not(): self; public function withTimeout(int $timeoutMs): self; diff --git a/src/Testing/PlaywrightTestCaseTrait.php b/src/Testing/PlaywrightTestCaseTrait.php index 7de60d3..959b23c 100644 --- a/src/Testing/PlaywrightTestCaseTrait.php +++ b/src/Testing/PlaywrightTestCaseTrait.php @@ -124,7 +124,31 @@ protected function assertElementExists(string $selector): void protected function expect(LocatorInterface|PageInterface $subject): ExpectInterface { - return new ExpectDecorator(new Expect($subject, $this->context->tracing()), $this); + return new ExpectDecorator(new Expect($subject, $this->context->tracing(), $this->snapshotDirectory(), $this->testOutputDirectory()), $this); + } + + /** + * Where a failing toHaveScreenshot() leaves its expected, actual and diff + * images: one directory per test, as Playwright JS does under + * test-results/. Override to write them elsewhere. + */ + protected function testOutputDirectory(): string + { + $class = (new \ReflectionClass($this))->getShortName(); + + return getcwd().'/test-failures/'.self::sanitizeFileName($class.'-'.$this->resolveTestName()); + } + + /** + * Where toHaveScreenshot() keeps baselines: next to the test file, as in. + * + * @playwright/test. Override to store them elsewhere. + */ + protected function snapshotDirectory(): string + { + $file = (new \ReflectionClass($this))->getFileName(); + + return false === $file ? getcwd().'/snapshots' : $file.'-snapshots'; } private function resolveLogger(LoggerInterface $logger): LoggerInterface diff --git a/tests/Functional/Screenshot/ToHaveScreenshotTest.php b/tests/Functional/Screenshot/ToHaveScreenshotTest.php new file mode 100644 index 0000000..9c78968 --- /dev/null +++ b/tests/Functional/Screenshot/ToHaveScreenshotTest.php @@ -0,0 +1,339 @@ +dir = sys_get_temp_dir().'/pw_to_have_screenshot_'.uniqid(); + unset($_SERVER[ScreenshotExpectation::UPDATE_ENV]); + } + + protected function tearDown(): void + { + unset($_SERVER[ScreenshotExpectation::UPDATE_ENV]); + $this->remove($this->dir); + foreach (glob(getcwd().'/test-failures/ToHaveScreenshotTest-*') ?: [] as $dir) { + $this->remove($dir); + } + + parent::tearDown(); + } + + protected function snapshotDirectory(): string + { + return $this->dir; + } + + public function testFirstRunWritesTheBaselineThenMatches(): void + { + $this->card('#818cf8'); + $options = new ToHaveScreenshotOptions(mask: [$this->page->locator('.time')]); + + try { + $this->expect($this->page)->toHaveScreenshot('page.png', $options); + $this->fail('The first run must fail after writing the baseline.'); + } catch (AssertionException $e) { + $this->assertStringContainsString('writing actual', $e->getMessage()); + } + + $baseline = $this->dir.'/page-'.$this->variant().'.png'; + $this->assertFileExists($baseline); + $this->assertSame("\x89PNG", substr((string) file_get_contents($baseline), 0, 4)); + + $this->card('#818cf8'); + $this->expect($this->page)->toHaveScreenshot('page.png', $options); + } + + public function testTimestampsAndInfiniteAnimationsAreStabilized(): void + { + $this->card('#818cf8'); + $card = $this->page->locator('#card'); + $options = new ToHaveScreenshotOptions(mask: [$this->page->locator('.time')]); + $this->record(fn () => $this->expect($card)->toHaveScreenshot('card.png', $options)); + + usleep(20_000); + $this->card('#818cf8'); + $this->expect($card)->toHaveScreenshot('card.png', $options); + } + + public function testChangedElementFailsAndKeepsTheImagesUntilItPassesAgain(): void + { + $this->card('#818cf8', 'Hello'); + $card = $this->page->locator('#card'); + $options = new ToHaveScreenshotOptions(mask: [$this->page->locator('.time')]); + $this->record(fn () => $this->expect($card)->toHaveScreenshot('card.png', $options)); + + $this->card('#818cf8', 'Hello world'); + + try { + $this->expect($card)->toHaveScreenshot('card.png', $options); + $this->fail('A changed element must fail the assertion.'); + } catch (AssertionException $e) { + $this->assertStringContainsString('Screenshot of the locator does not match', $e->getMessage()); + $this->assertStringContainsString('pixels', $e->getMessage()); + } + + $out = $this->testOutputDirectory().'-chromium/card'; + $this->assertStringEndsWith('/test-failures/ToHaveScreenshotTest-testChangedElementFailsAndKeepsTheImagesUntilItPassesAgain-chromium/card', $out); + $this->assertFileExists($out.'-expected.png'); + $this->assertFileExists($out.'-actual.png'); + $this->assertFileExists($out.'-diff.png'); + + $this->card('#818cf8', 'Hello'); + $this->expect($card)->toHaveScreenshot('card.png', $options); + + $this->assertFileDoesNotExist($out.'-expected.png'); + $this->assertFileDoesNotExist($out.'-actual.png'); + $this->assertFileDoesNotExist($out.'-diff.png'); + } + + public function testDefaultThresholdToleratesCloseShadesOfOneHue(): void + { + $this->card('#818cf8'); + $card = $this->page->locator('#card'); + $options = new ToHaveScreenshotOptions(mask: [$this->page->locator('.time')]); + $this->record(fn () => $this->expect($card)->toHaveScreenshot('card.png', $options)); + + $this->card('#6366f1'); + $this->expect($card)->toHaveScreenshot('card.png', $options); + + $this->expectException(AssertionException::class); + $this->expect($card)->toHaveScreenshot('card.png', new ToHaveScreenshotOptions(threshold: 0.1, mask: [$this->page->locator('.time')])); + } + + public function testSsimComparatorIsAvailable(): void + { + $this->card('#818cf8'); + $card = $this->page->locator('#card'); + $options = new ToHaveScreenshotOptions(comparator: 'ssim-cie94', mask: [$this->page->locator('.time')]); + $this->record(fn () => $this->expect($card)->toHaveScreenshot('card.png', $options)); + + $this->card('#6366f1'); + + $this->expectException(AssertionException::class); + $this->expect($card)->toHaveScreenshot('card.png', $options); + } + + public function testChangedModeAcceptsTheNewScreenshot(): void + { + $this->card('#818cf8', 'Hello'); + $card = $this->page->locator('#card'); + $options = new ToHaveScreenshotOptions(mask: [$this->page->locator('.time')]); + $this->record(fn () => $this->expect($card)->toHaveScreenshot('card.png', $options)); + $before = file_get_contents($this->dir.'/card-'.$this->variant().'.png'); + + $this->card('#818cf8', 'Hello world'); + $_SERVER[ScreenshotExpectation::UPDATE_ENV] = 'changed'; + $this->expect($card)->toHaveScreenshot('card.png', $options); + unset($_SERVER[ScreenshotExpectation::UPDATE_ENV]); + + $this->assertNotSame($before, file_get_contents($this->dir.'/card-'.$this->variant().'.png')); + $this->expect($card)->toHaveScreenshot('card.png', $options); + } + + public function testWebpBaselineIsLosslessAndComparedAsWebp(): void + { + $this->card('#818cf8', 'Hello'); + $card = $this->page->locator('#card'); + $options = new ToHaveScreenshotOptions(mask: [$this->page->locator('.time')]); + $this->record(fn () => $this->expect($card)->toHaveScreenshot('card.webp', $options)); + + $baseline = (string) file_get_contents($this->dir.'/card-'.$this->variant().'.webp'); + $this->assertSame('WEBP', substr($baseline, 8, 4)); + + $this->card('#818cf8', 'Hello'); + $this->expect($card)->toHaveScreenshot('card.webp', $options); + + $this->card('#818cf8', 'Hello world'); + try { + $this->expect($card)->toHaveScreenshot('card.webp', $options); + $this->fail('A changed element must fail the assertion.'); + } catch (AssertionException) { + } + + $out = $this->testOutputDirectory().'-chromium/card'; + $this->assertSame('WEBP', substr((string) file_get_contents($out.'-actual.webp'), 8, 4)); + $this->assertSame("\x89PNG", substr((string) file_get_contents($out.'-diff.png'), 0, 4)); + } + + public function testFullPageCapturesBeyondTheViewport(): void + { + $this->page->setViewportSize(800, 600); + $this->page->setContent('
'); + + $this->record(fn () => $this->expect($this->page)->toHaveScreenshot('full.png', new ToHaveScreenshotOptions(fullPage: true))); + + $size = getimagesize($this->dir.'/full-'.$this->variant().'.png'); + $this->assertIsArray($size); + $this->assertSame([800, 2000], [$size[0], $size[1]]); + + $this->expect($this->page)->toHaveScreenshot('full.png', new ToHaveScreenshotOptions(fullPage: true)); + } + + public function testClipCapturesARegion(): void + { + $this->card('#818cf8'); + $clip = new ToHaveScreenshotOptions(clip: ['x' => 10, 'y' => 20, 'width' => 120, 'height' => 40]); + + $this->record(fn () => $this->expect($this->page)->toHaveScreenshot('region.png', $clip)); + + $size = getimagesize($this->dir.'/region-'.$this->variant().'.png'); + $this->assertIsArray($size); + $this->assertSame([120, 40], [$size[0], $size[1]]); + } + + public function testMaskReachesInsideAnIframe(): void + { + $frame = fn (string $time): string => sprintf( + '', + htmlspecialchars(sprintf('

Updated %s

', $time)), + ); + $options = fn (): ToHaveScreenshotOptions => new ToHaveScreenshotOptions( + mask: [$this->page->frameLocator('#preview')->locator('.time')], + ); + + $this->page->setContent($frame('10:00:00')); + $this->page->frameLocator('#preview')->locator('.time')->waitFor(); + $this->record(fn () => $this->expect($this->page->locator('#preview'))->toHaveScreenshot('frame.png', $options())); + + $this->page->setContent($frame('23:59:59')); + $this->page->frameLocator('#preview')->locator('.time')->waitFor(); + $this->expect($this->page->locator('#preview'))->toHaveScreenshot('frame.png', $options()); + + $this->expectException(AssertionException::class); + $this->expect($this->page->locator('#preview'))->toHaveScreenshot('frame.png'); + } + + public function testAllModeRewritesEvenAMatchingBaseline(): void + { + $this->card('#818cf8'); + $card = $this->page->locator('#card'); + $options = new ToHaveScreenshotOptions(mask: [$this->page->locator('.time')]); + $this->record(fn () => $this->expect($card)->toHaveScreenshot('card.png', $options)); + $baseline = $this->dir.'/card-'.$this->variant().'.png'; + file_put_contents($baseline, (string) file_get_contents($baseline)."\0"); + + $_SERVER[ScreenshotExpectation::UPDATE_ENV] = 'all'; + $this->expect($card)->toHaveScreenshot('card.png', $options); + + $this->assertStringEndsNotWith("\0", (string) file_get_contents($baseline)); + } + + public function testNoneModeNeverWritesABaseline(): void + { + $this->card('#818cf8'); + $_SERVER[ScreenshotExpectation::UPDATE_ENV] = 'none'; + + try { + $this->expect($this->page->locator('#card'))->toHaveScreenshot('card.png'); + $this->fail('A missing baseline must fail in none mode.'); + } catch (AssertionException $e) { + $this->assertStringContainsString('PLAYWRIGHT_UPDATE_SNAPSHOTS=missing', $e->getMessage()); + } + + $this->assertFileDoesNotExist($this->dir.'/card-'.$this->variant().'.png'); + } + + public function testFullPageIsRejectedOnAnElement(): void + { + $this->card('#818cf8'); + + $this->expectException(\InvalidArgumentException::class); + $this->expectExceptionMessage('The fullPage and clip options apply to a page screenshot'); + + $this->expect($this->page->locator('#card'))->toHaveScreenshot('card.png', new ToHaveScreenshotOptions(fullPage: true)); + } + + public function testCannotBeNegated(): void + { + $this->card('#818cf8'); + + $this->expectException(\LogicException::class); + $this->expectExceptionMessage('toHaveScreenshot() cannot be negated.'); + + $this->expect($this->page)->not()->toHaveScreenshot('page.png'); + } + + private function card(string $color, string $title = 'Hello'): void + { + $this->page->setContent(sprintf(<<<'HTML' + +
+

%s

+

%s

+
+
+ + + HTML, $color, $title, microtime())); + } + + /** + * Runs a first assertion that records the baseline, which fails by design. + */ + private function record(callable $assertion): void + { + try { + $assertion(); + } catch (AssertionException $e) { + $this->assertStringContainsString('writing actual', $e->getMessage()); + } + } + + /** + * The functional suite runs Chromium. + */ + private function variant(): string + { + return 'chromium-'.('Windows' === \PHP_OS_FAMILY ? 'win32' : strtolower(\PHP_OS_FAMILY)); + } + + private function remove(string $dir): void + { + if (!is_dir($dir)) { + return; + } + + foreach (glob($dir.'/*') ?: [] as $file) { + is_dir($file) ? $this->remove($file) : unlink($file); + } + rmdir($dir); + } +} diff --git a/tests/Unit/Assertions/Internal/ScreenshotExpectationTest.php b/tests/Unit/Assertions/Internal/ScreenshotExpectationTest.php new file mode 100644 index 0000000..9a9864e --- /dev/null +++ b/tests/Unit/Assertions/Internal/ScreenshotExpectationTest.php @@ -0,0 +1,434 @@ +}> */ + private array $calls = []; + + protected function setUp(): void + { + $this->dir = sys_get_temp_dir().'/pw_screenshot_expectation_'.uniqid(); + mkdir($this->dir); + $this->calls = []; + } + + protected function tearDown(): void + { + $iterator = new \RecursiveIteratorIterator( + new \RecursiveDirectoryIterator($this->dir, \FilesystemIterator::SKIP_DOTS), + \RecursiveIteratorIterator::CHILD_FIRST, + ); + foreach ($iterator as $file) { + $file->isDir() ? rmdir($file->getPathname()) : unlink($file->getPathname()); + } + rmdir($this->dir); + } + + public function testMissingBaselineIsWrittenAndFailsByDefault(): void + { + $path = $this->dir.'/snapshots/card.png'; + + try { + $this->expectation(new ScreenshotComparison(actual: 'stable'))->assert($path, new ToHaveScreenshotOptions(), 5000); + $this->fail('A missing baseline must fail the assertion.'); + } catch (AssertionException $e) { + $this->assertStringContainsString("doesn't exist at {$path}, writing actual", $e->getMessage()); + } + + $this->assertSame('stable', file_get_contents($path)); + $this->assertNull($this->calls[0]['expected']); + } + + #[DataProvider('provideUpdatingModes')] + public function testMissingBaselineIsWrittenAndPassesWhenUpdating(string $mode): void + { + $path = $this->dir.'/card.png'; + + $this->expectation(new ScreenshotComparison(actual: 'stable'), $mode)->assert($path, new ToHaveScreenshotOptions(), 5000); + + $this->assertSame('stable', file_get_contents($path)); + } + + /** + * @return iterable + */ + public static function provideUpdatingModes(): iterable + { + yield 'changed' => ['changed']; + yield 'all' => ['all']; + } + + public function testMissingBaselineIsNotWrittenInNoneMode(): void + { + $path = $this->dir.'/card.png'; + + $this->expectException(AssertionException::class); + $this->expectExceptionMessage('Run with PLAYWRIGHT_UPDATE_SNAPSHOTS=missing to write it.'); + + try { + $this->expectation(new ScreenshotComparison(actual: 'stable'), 'none')->assert($path, new ToHaveScreenshotOptions(), 5000); + } finally { + $this->assertFileDoesNotExist($path); + } + } + + public function testUnstablePageFailsWithoutWritingABaseline(): void + { + $path = $this->dir.'/card.png'; + $unstable = new ScreenshotComparison(actual: 'last', errorMessage: 'Failed to take two consecutive stable screenshots.', timedOut: true); + + $this->expectException(AssertionException::class); + $this->expectExceptionMessage('Failed to take a stable screenshot of the page: Failed to take two consecutive stable screenshots.'); + + try { + $this->expectation($unstable)->assert($path, new ToHaveScreenshotOptions(), 5000); + } finally { + $this->assertFileDoesNotExist($path); + } + } + + public function testMatchingScreenshotPassesAndSendsTheBaseline(): void + { + $path = $this->baseline('baseline'); + + $this->expectation(new ScreenshotComparison(actual: null))->assert($path, new ToHaveScreenshotOptions(threshold: 0.1), 1234); + + $this->assertSame('baseline', $this->calls[0]['expected']); + $this->assertSame(0.1, $this->calls[0]['options']['threshold']); + $this->assertSame(1234, $this->calls[0]['options']['timeout']); + $this->assertSame('png', $this->calls[0]['options']['type']); + } + + public function testOptionTimeoutWinsOverTheAssertionTimeout(): void + { + $path = $this->baseline('baseline'); + + $this->expectation(new ScreenshotComparison(actual: null))->assert($path, new ToHaveScreenshotOptions(timeoutMs: 42), 5000); + + $this->assertSame(42, $this->calls[0]['options']['timeout']); + } + + public function testMismatchKeepsExpectedActualAndDiff(): void + { + $path = $this->baseline('baseline'); + $mismatch = new ScreenshotComparison(actual: 'actual', diff: 'diff', errorMessage: '120 pixels (ratio 0.02 of all image pixels) are different.'); + + try { + $this->expectation($mismatch)->assert($path, new ToHaveScreenshotOptions(), 5000); + $this->fail('A mismatch must fail the assertion.'); + } catch (AssertionException $e) { + $this->assertStringContainsString("Screenshot of the page does not match {$path}.", $e->getMessage()); + $this->assertStringContainsString('120 pixels (ratio 0.02 of all image pixels) are different.', $e->getMessage()); + $this->assertStringContainsString('PLAYWRIGHT_UPDATE_SNAPSHOTS=changed', $e->getMessage()); + $this->assertSame($this->dir.'/out/card-actual.png', $e->actual); + } + + $this->assertSame('baseline', file_get_contents($this->dir.'/out/card-expected.png')); + $this->assertSame('actual', file_get_contents($this->dir.'/out/card-actual.png')); + $this->assertSame('diff', file_get_contents($this->dir.'/out/card-diff.png')); + $this->assertSame('baseline', file_get_contents($path)); + } + + public function testMatchRemovesTheImagesOfAnEarlierFailure(): void + { + $path = $this->baseline('baseline'); + $this->staleArtifacts(); + + $this->expectation(new ScreenshotComparison(actual: null))->assert($path, new ToHaveScreenshotOptions(), 5000); + + $this->assertDirectoryDoesNotExist($this->dir.'/out'); + } + + /** + * @param 'changed'|'all' $mode + */ + #[DataProvider('provideUpdatingModes')] + public function testRecordingRemovesTheImagesOfAnEarlierFailure(string $mode): void + { + $path = $this->baseline('baseline'); + $this->staleArtifacts(); + + $this->expectation(new ScreenshotComparison(actual: 'actual', errorMessage: 'changed' === $mode ? 'different' : null), $mode) + ->assert($path, new ToHaveScreenshotOptions(), 5000); + + $this->assertDirectoryDoesNotExist($this->dir.'/out'); + } + + public function testFailureKeepsTheImagesOfOtherBaselines(): void + { + $path = $this->baseline('baseline'); + mkdir($this->dir.'/out'); + file_put_contents($this->dir.'/out/other-actual.png', 'other'); + + $this->expectation(new ScreenshotComparison(actual: null))->assert($path, new ToHaveScreenshotOptions(), 5000); + + $this->assertFileExists($this->dir.'/out/other-actual.png'); + } + + public function testFailureImagesDropTheVariantOfTheBaselineName(): void + { + $path = $this->dir.'/card-chromium-linux.webp'; + file_put_contents($path, 'baseline'); + $mismatch = new ScreenshotComparison(actual: 'actual', diff: 'diff', errorMessage: 'different'); + + try { + $this->expectation($mismatch, variant: 'chromium-linux')->assert($path, new ToHaveScreenshotOptions(), 5000); + $this->fail('A mismatch must fail the assertion.'); + } catch (AssertionException) { + } + + $this->assertSame(['card-actual.webp', 'card-diff.png', 'card-expected.webp'], array_map('basename', glob($this->dir.'/out/*') ?: [])); + + $this->expectation(new ScreenshotComparison(actual: null), variant: 'chromium-linux')->assert($path, new ToHaveScreenshotOptions(), 5000); + + $this->assertDirectoryDoesNotExist($this->dir.'/out'); + } + + public function testFailureImagesKeepANameThatDoesNotEndWithTheVariant(): void + { + $path = $this->baseline('baseline'); + + try { + $this->expectation(new ScreenshotComparison(actual: 'actual', errorMessage: 'different'), variant: 'chromium-linux') + ->assert($path, new ToHaveScreenshotOptions(), 5000); + } catch (AssertionException) { + } + + $this->assertFileExists($this->dir.'/out/card-actual.png'); + } + + public function testMismatchUsesTheCustomMessage(): void + { + $path = $this->baseline('baseline'); + + $this->expectException(AssertionException::class); + $this->expectExceptionMessage('The card changed.'); + + $this->expectation(new ScreenshotComparison(actual: 'actual', errorMessage: 'different')) + ->assert($path, new ToHaveScreenshotOptions(message: 'The card changed.'), 5000); + } + + public function testChangedModeRewritesAMismatchingBaseline(): void + { + $path = $this->baseline('baseline'); + + $this->expectation(new ScreenshotComparison(actual: 'actual', errorMessage: 'different'), 'changed')->assert($path, new ToHaveScreenshotOptions(), 5000); + + $this->assertSame('actual', file_get_contents($path)); + } + + public function testChangedModeKeepsAMatchingBaseline(): void + { + $path = $this->baseline('baseline'); + + $this->expectation(new ScreenshotComparison(actual: null), 'changed')->assert($path, new ToHaveScreenshotOptions(), 5000); + + $this->assertSame('baseline', file_get_contents($path)); + } + + public function testAllModeRecordsWithoutComparing(): void + { + $path = $this->baseline('baseline'); + + $this->expectation(new ScreenshotComparison(actual: 'stable'), 'all')->assert($path, new ToHaveScreenshotOptions(), 5000); + + $this->assertSame('stable', file_get_contents($path)); + $this->assertNull($this->calls[0]['expected']); + } + + public function testAllModeNeverRecordsAnUnstableScreenshot(): void + { + $path = $this->baseline('baseline'); + + $this->expectException(AssertionException::class); + $this->expectExceptionMessage('Failed to take a stable screenshot'); + + try { + $this->expectation(new ScreenshotComparison(actual: 'moving', errorMessage: 'unstable', timedOut: true), 'all') + ->assert($path, new ToHaveScreenshotOptions(), 5000); + } finally { + $this->assertSame('baseline', file_get_contents($path)); + } + } + + public function testMismatchFailsInNoneMode(): void + { + $path = $this->baseline('baseline'); + + $this->expectException(AssertionException::class); + + try { + $this->expectation(new ScreenshotComparison(actual: 'actual', errorMessage: 'different'), 'none')->assert($path, new ToHaveScreenshotOptions(), 5000); + } finally { + $this->assertSame('baseline', file_get_contents($path)); + } + } + + public function testTimedOutMismatchNeverRewritesTheBaseline(): void + { + $path = $this->baseline('baseline'); + + $this->expectException(AssertionException::class); + + try { + $this->expectation(new ScreenshotComparison(actual: 'moving', errorMessage: 'unstable', timedOut: true), 'changed') + ->assert($path, new ToHaveScreenshotOptions(), 5000); + } finally { + $this->assertSame('baseline', file_get_contents($path)); + } + } + + public function testWebpBaselineRequestsWebpAndKeepsWebpArtifacts(): void + { + $path = $this->dir.'/card.webp'; + file_put_contents($path, 'baseline'); + + try { + $this->expectation(new ScreenshotComparison(actual: 'actual', diff: 'diff', errorMessage: 'different'))->assert($path, new ToHaveScreenshotOptions(), 5000); + $this->fail('A mismatch must fail the assertion.'); + } catch (AssertionException) { + } + + $this->assertSame('webp', $this->calls[0]['options']['type']); + $this->assertFileExists($this->dir.'/out/card-expected.webp'); + $this->assertFileExists($this->dir.'/out/card-actual.webp'); + $this->assertFileExists($this->dir.'/out/card-diff.png'); + } + + public function testJpegBaselineIsRejected(): void + { + $this->expectException(\InvalidArgumentException::class); + $this->expectExceptionMessage('must be a .png or .webp file.'); + + $this->expectation(new ScreenshotComparison(actual: 'stable'))->assert($this->dir.'/card.jpg', new ToHaveScreenshotOptions(), 5000); + } + + public function testUnstablePageWithABaselineKeepsTheLastScreenshot(): void + { + $path = $this->baseline('baseline'); + + try { + $this->expectation(new ScreenshotComparison(actual: 'moving', errorMessage: 'Failed to take two consecutive stable screenshots.', timedOut: true)) + ->assert($path, new ToHaveScreenshotOptions(), 5000); + $this->fail('An unstable page must fail the assertion.'); + } catch (AssertionException $e) { + $this->assertStringContainsString('Failed to take two consecutive stable screenshots.', $e->getMessage()); + } + + $this->assertSame('baseline', file_get_contents($path)); + $this->assertSame('baseline', file_get_contents($this->dir.'/out/card-expected.png')); + $this->assertSame('moving', file_get_contents($this->dir.'/out/card-actual.png')); + } + + public function testUnreadableBaselineIsReported(): void + { + if (0 === posix_geteuid()) { + $this->markTestSkipped('Root reads files whatever their permissions.'); + } + + $path = $this->baseline('baseline'); + chmod($path, 0o000); + + $this->expectException(\RuntimeException::class); + $this->expectExceptionMessage(sprintf('Unable to read the snapshot "%s".', $path)); + + try { + $this->expectation(new ScreenshotComparison(actual: null))->assert($path, new ToHaveScreenshotOptions(), 5000); + } finally { + chmod($path, 0o644); + } + } + + public function testUncreatableOutputDirectoryIsReported(): void + { + $path = $this->baseline('baseline'); + file_put_contents($this->dir.'/file', 'not a directory'); + + $expectation = new ScreenshotExpectation( + static fn (): ScreenshotComparison => new ScreenshotComparison(actual: 'actual', errorMessage: 'different'), + 'page', + 'none', + $this->dir.'/file/out', + ); + + $this->expectException(\RuntimeException::class); + $this->expectExceptionMessage(sprintf('Directory "%s/file/out" was not created.', $this->dir)); + + $expectation->assert($path, new ToHaveScreenshotOptions(), 5000); + } + + public function testUnwritableBaselineIsReported(): void + { + $path = $this->dir.'/card.png'; + mkdir($path); + + $this->expectException(\RuntimeException::class); + $this->expectExceptionMessage(sprintf('Unable to write "%s".', $path)); + + $this->expectation(new ScreenshotComparison(actual: 'stable'), 'changed')->assert($path, new ToHaveScreenshotOptions(), 5000); + } + + public function testUnknownUpdateModeIsRejected(): void + { + $this->expectException(\InvalidArgumentException::class); + $this->expectExceptionMessage('PLAYWRIGHT_UPDATE_SNAPSHOTS must be one of "none", "missing", "changed", "all", "yes" given.'); + + $this->expectation(new ScreenshotComparison(actual: 'stable'), 'yes')->assert($this->dir.'/card.png', new ToHaveScreenshotOptions(), 5000); + } + + private function staleArtifacts(): void + { + mkdir($this->dir.'/out'); + foreach (['card-expected.png', 'card-actual.png', 'card-diff.png'] as $file) { + file_put_contents($this->dir.'/out/'.$file, 'stale'); + } + } + + private function baseline(string $contents): string + { + $path = $this->dir.'/card.png'; + file_put_contents($path, $contents); + + return $path; + } + + private function expectation(ScreenshotComparison $result, ?string $mode = null, ?string $variant = null): ScreenshotExpectation + { + return new ScreenshotExpectation( + function (?string $expected, array $options) use ($result): ScreenshotComparison { + $this->calls[] = ['expected' => $expected, 'options' => $options]; + + return $result; + }, + 'page', + $mode, + $this->dir.'/out', + $variant, + ); + } +} diff --git a/tests/Unit/Assertions/LocatorAssertionsTest.php b/tests/Unit/Assertions/LocatorAssertionsTest.php index 56edce2..d660f22 100644 --- a/tests/Unit/Assertions/LocatorAssertionsTest.php +++ b/tests/Unit/Assertions/LocatorAssertionsTest.php @@ -15,11 +15,17 @@ namespace Playwright\Tests\Unit\Assertions; use PHPUnit\Framework\Attributes\CoversClass; +use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\TestCase; use Playwright\Assertions\AssertionOptions; use Playwright\Assertions\Failure\AssertionException; +use Playwright\Assertions\Internal\ScreenshotExpectation; use Playwright\Assertions\LocatorAssertions; +use Playwright\Assertions\Options\ToHaveScreenshotOptions; +use Playwright\Locator\Locator; use Playwright\Locator\LocatorInterface; +use Playwright\Tracing\TracingInterface; +use Playwright\Transport\TransportInterface; #[CoversClass(LocatorAssertions::class)] final class LocatorAssertionsTest extends TestCase @@ -364,4 +370,125 @@ public function testToBeInViewportRejectsANegativeRatio(): void (new LocatorAssertions($this->createMock(LocatorInterface::class))) ->toBeInViewport(new AssertionOptions(ratio: -0.1)); } + + /** @var list */ + private array $dirs = []; + + public function testToHaveScreenshotNeedsAPlaywrightLocator(): void + { + $this->expectException(\LogicException::class); + $this->expectExceptionMessage('toHaveScreenshot() needs a Playwright\Locator\Locator created by Playwright PHP'); + + (new LocatorAssertions($this->createStub(LocatorInterface::class)))->toHaveScreenshot('/tmp/card.png'); + } + + /** + * @param array $arguments + */ + #[DataProvider('providePageOnlyOptions')] + public function testToHaveScreenshotRejectsPageOnlyOptions(array $arguments): void + { + $transport = $this->createMock(TransportInterface::class); + $transport->expects($this->never())->method('send'); + + $this->expectException(\InvalidArgumentException::class); + $this->expectExceptionMessage('The fullPage and clip options apply to a page screenshot'); + + (new LocatorAssertions(new Locator($transport, 'page1', '#card')))->toHaveScreenshot('/tmp/card.png', new ToHaveScreenshotOptions(...$arguments)); + } + + /** + * @return iterable}> + */ + public static function providePageOnlyOptions(): iterable + { + yield 'fullPage' => [['fullPage' => true]]; + yield 'clip' => [['clip' => ['x' => 0, 'y' => 0, 'width' => 10, 'height' => 10]]]; + } + + public function testToHaveScreenshotComparesWithTheBaselineInsideATracingGroup(): void + { + $path = $this->snapshotDir().'/card.png'; + file_put_contents($path, 'baseline'); + + $transport = $this->createMock(TransportInterface::class); + $transport->expects($this->once()) + ->method('send') + ->with($this->callback(static fn (array $payload): bool => 'locator.expectScreenshot' === $payload['action'] + && base64_encode('baseline') === $payload['expected'] + && 1234 === $payload['options']['timeout'] + && 'png' === $payload['options']['type'])) + ->willReturn(['errorMessage' => null]); + + $tracing = $this->createMock(TracingInterface::class); + $tracing->expects($this->once())->method('group')->with('expect(#card).toHaveScreenshot'); + $tracing->expects($this->once())->method('groupEnd'); + + $assertions = new LocatorAssertions(new Locator($transport, 'page1', '#card'), $tracing); + + $this->assertSame($assertions, $assertions->withTimeout(1234)->toHaveScreenshot($path)); + } + + public function testToHaveScreenshotWritesFailureImagesInTheGivenDirectory(): void + { + $dir = $this->snapshotDir(); + $path = $dir.'/card.png'; + file_put_contents($path, 'baseline'); + $_SERVER[ScreenshotExpectation::UPDATE_ENV] = 'none'; + + $transport = $this->createStub(TransportInterface::class); + $transport->method('send')->willReturn(['actual' => base64_encode('actual'), 'diff' => base64_encode('diff'), 'errorMessage' => 'different']); + + try { + (new LocatorAssertions(new Locator($transport, 'page1', '#card'), null, $dir.'/out'))->toHaveScreenshot($path); + $this->fail('A mismatch must fail the assertion.'); + } catch (AssertionException) { + } + + $this->assertSame('actual', file_get_contents($dir.'/out/card-actual.png')); + $this->assertSame('diff', file_get_contents($dir.'/out/card-diff.png')); + foreach (glob($dir.'/out/*') ?: [] as $file) { + unlink($file); + } + rmdir($dir.'/out'); + } + + public function testToHaveScreenshotCannotBeNegatedAndResetsTheNegation(): void + { + $path = $this->snapshotDir().'/card.png'; + file_put_contents($path, 'baseline'); + + $transport = $this->createMock(TransportInterface::class); + $transport->expects($this->once())->method('send')->willReturn(['errorMessage' => null]); + $assertions = new LocatorAssertions(new Locator($transport, 'page1', '#card')); + + try { + $assertions->not()->toHaveScreenshot($path); + $this->fail('A negated screenshot assertion must throw.'); + } catch (\LogicException $e) { + $this->assertSame('toHaveScreenshot() cannot be negated.', $e->getMessage()); + } + + $this->assertSame($assertions, $assertions->toHaveScreenshot($path)); + } + + private function snapshotDir(): string + { + $dir = sys_get_temp_dir().'/pw-assertions-'.bin2hex(random_bytes(6)); + mkdir($dir); + $this->dirs[] = $dir; + + return $dir; + } + + protected function tearDown(): void + { + unset($_SERVER[ScreenshotExpectation::UPDATE_ENV]); + foreach ($this->dirs as $dir) { + foreach (glob($dir.'/*') ?: [] as $file) { + unlink($file); + } + rmdir($dir); + } + } } diff --git a/tests/Unit/Assertions/Options/ToHaveScreenshotOptionsTest.php b/tests/Unit/Assertions/Options/ToHaveScreenshotOptionsTest.php new file mode 100644 index 0000000..bf9e665 --- /dev/null +++ b/tests/Unit/Assertions/Options/ToHaveScreenshotOptionsTest.php @@ -0,0 +1,145 @@ +assertSame([ + 'threshold' => 0.2, + 'comparator' => 'pixelmatch', + 'maskColor' => '#FF00FF', + 'fullPage' => false, + 'omitBackground' => false, + 'animations' => 'disabled', + 'caret' => 'hide', + 'scale' => 'css', + ], (new ToHaveScreenshotOptions())->toArray()); + } + + public function testToleranceHasNoDefault(): void + { + $options = new ToHaveScreenshotOptions(); + + $this->assertNull($options->maxDiffPixels); + $this->assertNull($options->maxDiffPixelRatio); + $this->assertNull($options->timeoutMs); + } + + public function testSendsTheValuesSet(): void + { + $options = new ToHaveScreenshotOptions( + threshold: 0.1, + maxDiffPixels: 10, + maxDiffPixelRatio: 0.01, + comparator: ToHaveScreenshotOptions::COMPARATOR_SSIM_CIE94, + style: '.ad { display: none }', + fullPage: true, + clip: ['x' => 0, 'y' => 0, 'width' => 320, 'height' => 200], + omitBackground: true, + animations: 'allow', + caret: 'initial', + scale: 'device', + ); + + $this->assertSame([ + 'threshold' => 0.1, + 'maxDiffPixels' => 10, + 'maxDiffPixelRatio' => 0.01, + 'comparator' => 'ssim-cie94', + 'maskColor' => '#FF00FF', + 'style' => '.ad { display: none }', + 'fullPage' => true, + 'clip' => ['x' => 0, 'y' => 0, 'width' => 320, 'height' => 200], + 'omitBackground' => true, + 'animations' => 'allow', + 'caret' => 'initial', + 'scale' => 'device', + ], $options->toArray()); + } + + public function testTimeoutAndMessageStayOnThePhpSide(): void + { + $options = (new ToHaveScreenshotOptions(timeoutMs: 1000, message: 'The card changed.'))->toArray(); + + $this->assertArrayNotHasKey('timeoutMs', $options); + $this->assertArrayNotHasKey('timeout', $options); + $this->assertArrayNotHasKey('message', $options); + } + + public function testSerializesMaskLocatorsAsSelectors(): void + { + $locator = $this->createStub(LocatorInterface::class); + $locator->method('getSelector')->willReturn('.timestamp'); + + $options = new ToHaveScreenshotOptions(mask: [$locator], maskColor: 'black'); + + $this->assertSame('black', $options->toArray()['maskColor']); + $this->assertSame([['selector' => '.timestamp']], $options->toArray()['mask']); + } + + public function testMaskLocatorsInAFrameKeepTheirFrame(): void + { + $locator = new Locator($this->createStub(TransportInterface::class), 'page1', '.date', 'iframe#preview'); + + $options = new ToHaveScreenshotOptions(mask: [$locator]); + + $this->assertSame([['selector' => '.date', 'frameSelector' => 'iframe#preview']], $options->toArray()['mask']); + } + + /** + * @param array $arguments + */ + #[DataProvider('provideInvalidOptions')] + public function testRejectsInvalidOptions(array $arguments, string $message): void + { + $this->expectException(\InvalidArgumentException::class); + $this->expectExceptionMessage($message); + + new ToHaveScreenshotOptions(...$arguments); + } + + /** + * @return iterable, string}> + */ + public static function provideInvalidOptions(): iterable + { + yield 'threshold above 1' => [['threshold' => 2.0], 'The threshold option must be between 0 and 1, 2 given.']; + yield 'negative threshold' => [['threshold' => -0.1], 'The threshold option must be between 0 and 1, -0.1 given.']; + yield 'ratio above 1' => [['maxDiffPixelRatio' => 1.5], 'The maxDiffPixelRatio option must be between 0 and 1, 1.5 given.']; + yield 'negative pixels' => [['maxDiffPixels' => -1], 'The maxDiffPixels option must be 0 or more, -1 given.']; + yield 'negative timeout' => [['timeoutMs' => -1], 'The timeoutMs option must be 0 or more, -1 given.']; + yield 'unknown comparator' => [['comparator' => 'ssim'], 'The comparator option must be one of "pixelmatch", "ssim-cie94", "ssim" given.']; + yield 'unknown animations' => [['animations' => 'none'], 'The animations option must be one of "disabled", "allow", "none" given.']; + yield 'unknown caret' => [['caret' => 'hidden'], 'The caret option must be one of "hide", "initial", "hidden" given.']; + yield 'unknown scale' => [['scale' => 'px'], 'The scale option must be one of "css", "device", "px" given.']; + yield 'empty mask color' => [['maskColor' => ' '], 'The maskColor option must be a CSS color, an empty string given.']; + yield 'selector in mask' => [['mask' => ['.date']], 'The mask option expects Playwright\Locator\LocatorInterface instances, string given at index 0.']; + yield 'clip without height' => [['clip' => ['x' => 0, 'y' => 0, 'width' => 10]], 'The clip option needs numeric "x", "y", "width" and "height" keys, "height" is missing or not a number.']; + yield 'clip with a string' => [['clip' => ['x' => '0', 'y' => 0, 'width' => 10, 'height' => 10]], '"x" is missing or not a number.']; + yield 'empty clip' => [['clip' => ['x' => 0, 'y' => 0, 'width' => 0, 'height' => 10]], 'The clip option needs a width and a height greater than 0.']; + yield 'clip with an extra key' => [['clip' => ['x' => 0, 'y' => 0, 'width' => 10, 'height' => 10, 'scale' => 2]], 'The clip option only accepts "x", "y", "width" and "height" keys, "x", "y", "width", "height", "scale" given.']; + } +} diff --git a/tests/Unit/Assertions/PageAssertionsTest.php b/tests/Unit/Assertions/PageAssertionsTest.php index 1f59720..9fc307e 100644 --- a/tests/Unit/Assertions/PageAssertionsTest.php +++ b/tests/Unit/Assertions/PageAssertionsTest.php @@ -18,10 +18,16 @@ use PHPUnit\Framework\TestCase; use Playwright\Assertions\AssertionOptions; use Playwright\Assertions\Failure\AssertionException; +use Playwright\Assertions\Internal\ScreenshotExpectation; +use Playwright\Assertions\Options\ToHaveScreenshotOptions; use Playwright\Assertions\PageAssertions; +use Playwright\Browser\BrowserContextInterface; +use Playwright\Configuration\PlaywrightConfig; use Playwright\Locator\LocatorInterface; +use Playwright\Page\Page; use Playwright\Page\PageInterface; use Playwright\Tracing\TracingInterface; +use Playwright\Transport\TransportInterface; #[CoversClass(PageAssertions::class)] final class PageAssertionsTest extends TestCase @@ -128,4 +134,87 @@ private function pageSnapshotting(string $snapshot): PageInterface return $page; } + + /** @var list */ + private array $dirs = []; + + public function testToHaveScreenshotNeedsAPlaywrightPage(): void + { + $this->expectException(\LogicException::class); + $this->expectExceptionMessage('toHaveScreenshot() needs a Playwright\Page\Page created by Playwright PHP'); + + (new PageAssertions($this->createStub(PageInterface::class)))->toHaveScreenshot('/tmp/page.png'); + } + + public function testToHaveScreenshotSendsPageOptionsInsideATracingGroup(): void + { + $path = $this->snapshotDir().'/page.png'; + file_put_contents($path, 'baseline'); + + $transport = $this->createMock(TransportInterface::class); + $transport->expects($this->once()) + ->method('send') + ->with($this->callback(static fn (array $payload): bool => 'page.expectScreenshot' === $payload['action'] + && true === $payload['options']['fullPage'] + && ['x' => 0, 'y' => 0, 'width' => 320, 'height' => 200] === $payload['options']['clip'])) + ->willReturn(['errorMessage' => null]); + + $tracing = $this->createMock(TracingInterface::class); + $tracing->expects($this->once())->method('group')->with('expect(page).toHaveScreenshot'); + $tracing->expects($this->once())->method('groupEnd'); + + $page = new Page($transport, $this->createStub(BrowserContextInterface::class), 'page-id', new PlaywrightConfig()); + $assertions = new PageAssertions($page, $tracing); + + $this->assertSame($assertions, $assertions->toHaveScreenshot($path, new ToHaveScreenshotOptions( + fullPage: true, + clip: ['x' => 0, 'y' => 0, 'width' => 320, 'height' => 200], + ))); + } + + public function testToHaveScreenshotClosesTheTracingGroupOnFailure(): void + { + $path = $this->snapshotDir().'/page.png'; + file_put_contents($path, 'baseline'); + $_SERVER[ScreenshotExpectation::UPDATE_ENV] = 'none'; + + $transport = $this->createStub(TransportInterface::class); + $transport->method('send')->willReturn(['actual' => base64_encode('actual'), 'errorMessage' => 'different']); + + $tracing = $this->createMock(TracingInterface::class); + $tracing->expects($this->once())->method('groupEnd'); + + $page = new Page($transport, $this->createStub(BrowserContextInterface::class), 'page-id', new PlaywrightConfig()); + + $this->expectException(AssertionException::class); + + try { + (new PageAssertions($page, $tracing))->toHaveScreenshot($path); + } finally { + foreach (glob(getcwd().'/test-failures/snapshots/page-*') ?: [] as $file) { + unlink($file); + } + @rmdir(getcwd().'/test-failures/snapshots'); + } + } + + private function snapshotDir(): string + { + $dir = sys_get_temp_dir().'/pw-assertions-'.bin2hex(random_bytes(6)); + mkdir($dir); + $this->dirs[] = $dir; + + return $dir; + } + + protected function tearDown(): void + { + unset($_SERVER[ScreenshotExpectation::UPDATE_ENV]); + foreach ($this->dirs as $dir) { + foreach (glob($dir.'/*') ?: [] as $file) { + unlink($file); + } + rmdir($dir); + } + } } diff --git a/tests/Unit/Testing/ExpectFactoryTest.php b/tests/Unit/Testing/ExpectFactoryTest.php index bb0c072..8e93db1 100644 --- a/tests/Unit/Testing/ExpectFactoryTest.php +++ b/tests/Unit/Testing/ExpectFactoryTest.php @@ -18,14 +18,22 @@ use PHPUnit\Framework\Attributes\CoversFunction; use PHPUnit\Framework\Attributes\CoversTrait; use PHPUnit\Framework\TestCase; +use Playwright\Assertions\Failure\AssertionException; +use Playwright\Assertions\Internal\ScreenshotExpectation; use Playwright\Browser\BrowserContextInterface; +use Playwright\Browser\BrowserInterface; +use Playwright\Browser\BrowserType; +use Playwright\Configuration\PlaywrightConfig; +use Playwright\Locator\Locator; use Playwright\Locator\LocatorInterface; +use Playwright\Page\Page; use Playwright\Page\PageInterface; use Playwright\Testing\Expect; use Playwright\Testing\ExpectDecorator; use Playwright\Testing\ExpectInterface; use Playwright\Testing\PlaywrightTestCaseTrait; use Playwright\Tracing\TracingInterface; +use Playwright\Transport\TransportInterface; use function Playwright\Testing\expect; @@ -127,4 +135,273 @@ public function testNegationDoesNotLeakToTheNextAssertion(): void $expect->not()->toBeVisible(); $expect->toBeVisible(); } + + /** @var list */ + private array $dirs = []; + + public function testRelativeScreenshotNamesGetThePlatformSuffix(): void + { + $dir = $this->snapshotDir(); + + $this->recordBaseline(new Expect($this->pageReturning('stable'), null, $dir), 'card.png'); + + $this->assertSame('stable', file_get_contents($dir.'/card-'.self::platform().'.png')); + } + + public function testScreenshotNamesWithoutExtensionArePng(): void + { + $dir = $this->snapshotDir(); + + $this->recordBaseline(new Expect($this->pageReturning('stable'), null, $dir), 'card'); + + $this->assertFileExists($dir.'/card-'.self::platform().'.png'); + } + + public function testScreenshotNamesCarryTheBrowserOfThePage(): void + { + $dir = $this->snapshotDir(); + + $this->recordBaseline(new Expect($this->pageReturning('stable', browser: BrowserType::FIREFOX), null, $dir), 'card.png'); + + $this->assertFileExists($dir.'/card-firefox-'.self::platform().'.png'); + } + + public function testLocatorScreenshotNamesCarryTheBrowserOfTheirPage(): void + { + $dir = $this->snapshotDir(); + $transport = $this->transportReturning('stable', 'webp', 'locator.expectScreenshot'); + $page = new Page($transport, $this->contextOf(BrowserType::WEBKIT), 'page-id', new PlaywrightConfig()); + $locator = new Locator($transport, 'page-id', '#card', null, null, [], $page); + + $this->recordBaseline(new Expect($locator, null, $dir), 'card.webp'); + + $this->assertFileExists($dir.'/card-webkit-'.self::platform().'.webp'); + } + + public function testWebpScreenshotNamesKeepTheirExtension(): void + { + $dir = $this->snapshotDir(); + + $this->recordBaseline(new Expect($this->pageReturning('stable', type: 'webp'), null, $dir), 'card.webp'); + + $this->assertFileExists($dir.'/card-'.self::platform().'.webp'); + } + + public function testAbsoluteScreenshotPathsAreUsedAsIs(): void + { + $path = $this->snapshotDir().'/exact.png'; + file_put_contents($path, 'baseline'); + + (new Expect($this->pageReturning(null), null, '/elsewhere'))->toHaveScreenshot($path); + + $this->assertSame('baseline', file_get_contents($path)); + } + + public function testWindowsScreenshotPathsAreUsedAsIs(): void + { + $_SERVER[ScreenshotExpectation::UPDATE_ENV] = 'none'; + + $this->expectException(AssertionException::class); + $this->expectExceptionMessage('A snapshot doesn\'t exist at C:\\snapshots\\card.png.'); + + (new Expect($this->pageReturning('stable'), null, '/elsewhere'))->toHaveScreenshot('C:\\snapshots\\card.png'); + } + + public function testScreenshotsWithoutSnapshotDirectoryResolveInTheWorkingDirectory(): void + { + $dir = $this->snapshotDir(); + $cwd = (string) getcwd(); + chdir($dir); + + try { + $this->recordBaseline(new Expect($this->pageReturning('stable')), 'card.png'); + } finally { + chdir($cwd); + } + + $this->assertFileExists($dir.'/card-'.self::platform().'.png'); + } + + public function testTheDecoratorCountsTheScreenshotAssertion(): void + { + $path = $this->snapshotDir().'/card.png'; + file_put_contents($path, 'baseline'); + + $harness = new class('harness') extends TestCase { + }; + + (new ExpectDecorator(new Expect($this->pageReturning(null)), $harness))->toHaveScreenshot($path); + + $this->assertSame(1, $harness->numberOfAssertionsPerformed()); + } + + public function testFailureImagesGoToADirectoryPerBrowser(): void + { + $dir = $this->snapshotDir(); + $baseline = $dir.'/card-firefox-'.self::platform().'.png'; + file_put_contents($baseline, 'baseline'); + $_SERVER[ScreenshotExpectation::UPDATE_ENV] = 'none'; + + $transport = $this->createStub(TransportInterface::class); + $transport->method('send')->willReturn(['actual' => base64_encode('actual'), 'diff' => base64_encode('diff'), 'errorMessage' => 'different']); + $page = new Page($transport, $this->contextOf(BrowserType::FIREFOX), 'page-id', new PlaywrightConfig()); + + try { + (new Expect($page, null, $dir, $dir.'/CardTest-testCard'))->toHaveScreenshot('card.png'); + $this->fail('A mismatch must fail the assertion.'); + } catch (AssertionException) { + } + + $out = $dir.'/CardTest-testCard-firefox'; + $this->assertSame(['card-actual.png', 'card-diff.png', 'card-expected.png'], array_map('basename', glob($out.'/*') ?: [])); + foreach (glob($out.'/*') ?: [] as $file) { + unlink($file); + } + rmdir($out); + } + + public function testALocatorWithoutPageKeepsThePlatformOnly(): void + { + $dir = $this->snapshotDir(); + $transport = $this->transportReturning('stable', 'png', 'locator.expectScreenshot'); + + $this->recordBaseline(new Expect(new Locator($transport, 'page-id', '#card'), null, $dir), 'card.png'); + + $this->assertFileExists($dir.'/card-'.self::platform().'.png'); + } + + public function testTheTraitStoresSnapshotsNextToTheTestFile(): void + { + $harness = new class('harness') extends TestCase { + use PlaywrightTestCaseTrait; + }; + + $this->assertSame(__FILE__.'-snapshots', (new \ReflectionMethod($harness, 'snapshotDirectory'))->invoke($harness)); + } + + public function testTheTraitGivesEachTestItsOwnOutputDirectory(): void + { + $first = new class('testCard') extends TestCase { + use PlaywrightTestCaseTrait; + }; + $second = new class('testButton') extends TestCase { + use PlaywrightTestCaseTrait; + }; + + $method = new \ReflectionMethod($first, 'testOutputDirectory'); + $firstDir = $method->invoke($first); + $secondDir = (new \ReflectionMethod($second, 'testOutputDirectory'))->invoke($second); + + $this->assertStringStartsWith(getcwd().'/test-failures/', $firstDir); + $this->assertStringEndsWith('-testCard', $firstDir); + $this->assertStringEndsWith('-testButton', $secondDir); + $this->assertDoesNotMatchRegularExpression('~[^A-Za-z0-9._-]~', basename($firstDir)); + } + + public function testTheTraitPassesTheSnapshotDirectoryToExpect(): void + { + $dir = $this->snapshotDir(); + $page = $this->pageReturning('stable'); + + $context = $this->createStub(BrowserContextInterface::class); + $context->method('tracing')->willReturn($this->createStub(TracingInterface::class)); + + $harness = new class('harness') extends TestCase { + use PlaywrightTestCaseTrait; + + public string $dir = ''; + + public function callExpect(BrowserContextInterface $context, PageInterface $subject): ExpectInterface + { + $this->context = $context; + + return $this->expect($subject); + } + + protected function snapshotDirectory(): string + { + return $this->dir; + } + }; + $harness->dir = $dir; + + try { + $harness->callExpect($context, $page)->toHaveScreenshot('card.png'); + } catch (AssertionException) { + } + + $this->assertFileExists($dir.'/card-'.self::platform().'.png'); + } + + protected function tearDown(): void + { + unset($_SERVER[ScreenshotExpectation::UPDATE_ENV]); + foreach ($this->dirs as $dir) { + foreach (glob($dir.'/*') ?: [] as $file) { + unlink($file); + } + rmdir($dir); + } + } + + private function recordBaseline(Expect $expect, string $name): void + { + try { + $expect->toHaveScreenshot($name); + $this->fail('Recording a baseline must fail the assertion.'); + } catch (AssertionException $e) { + $this->assertStringContainsString('writing actual', $e->getMessage()); + } + } + + /** + * A page whose bridge returns $actual for every screenshot comparison. + */ + private function pageReturning(?string $actual, string $type = 'png', ?BrowserType $browser = null): Page + { + return new Page($this->transportReturning($actual, $type, 'page.expectScreenshot'), $this->contextOf($browser), 'page-id', new PlaywrightConfig()); + } + + private function transportReturning(?string $actual, string $type, string $action): TransportInterface + { + $transport = $this->createStub(TransportInterface::class); + $transport->method('send')->willReturnCallback(static function (array $payload) use ($actual, $type, $action): array { + if ($action !== $payload['action'] || $type !== $payload['options']['type']) { + throw new \UnexpectedValueException('Unexpected command: '.json_encode($payload)); + } + + return ['actual' => null === $actual ? null : base64_encode($actual), 'errorMessage' => null, 'timedOut' => false]; + }); + + return $transport; + } + + /** + * A context without browser stands for a persistent context. + */ + private function contextOf(?BrowserType $browser): BrowserContextInterface + { + $context = $this->createStub(BrowserContextInterface::class); + if (null !== $browser) { + $instance = $this->createStub(BrowserInterface::class); + $instance->method('browserType')->willReturn($browser); + $context->method('browser')->willReturn($instance); + } + + return $context; + } + + private function snapshotDir(): string + { + $dir = sys_get_temp_dir().'/pw-expect-'.bin2hex(random_bytes(6)); + mkdir($dir); + $this->dirs[] = $dir; + + return $dir; + } + + private static function platform(): string + { + return 'Windows' === \PHP_OS_FAMILY ? 'win32' : strtolower(\PHP_OS_FAMILY); + } } From 79e4e9bf86f319b340071ac1a5f463d01ac99c15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Andr=C3=A9?= Date: Sat, 26 Sep 2026 21:18:24 +0000 Subject: [PATCH 3/5] Document visual regression testing A guide on why and where to compare screenshots (a defensive test on the design system and precious pages), how baselines are named, updated and reviewed, failure images, PNG and WebP, options, headless mode and limits. --- docs/guide/assertions-reference.md | 2 + docs/guide/visual-regression.md | 233 +++++++++++++++++++++++++++++ 2 files changed, 235 insertions(+) create mode 100644 docs/guide/visual-regression.md diff --git a/docs/guide/assertions-reference.md b/docs/guide/assertions-reference.md index 7833f05..9f88ff1 100644 --- a/docs/guide/assertions-reference.md +++ b/docs/guide/assertions-reference.md @@ -80,6 +80,7 @@ These assertions are available when you pass a `Locator` to `expect()`. * **`toHaveId(string $id)`**: Asserts the element has the given ID. * **`toHaveClass(string|array $class)`**: Asserts the complete class list matches. * **`toHaveCount(int $count)`**: Asserts the locator resolves to a specific number of elements. +* **`toHaveScreenshot(string $name, ?ToHaveScreenshotOptions $options = null)`**: Compares a stable screenshot of the element with a baseline image. See [Visual Regression](visual-regression.md). ----- @@ -89,3 +90,4 @@ These assertions are available when you pass a `Page` object to `expect()`. * **`toHaveURL(string $url)`**: Asserts the page's current URL is a match. * **`toHaveTitle(string $title)`**: Asserts the page's title is a match. +* **`toHaveScreenshot(string $name, ?ToHaveScreenshotOptions $options = null)`**: Compares a stable screenshot of the page with a baseline image. See [Visual Regression](visual-regression.md). diff --git a/docs/guide/visual-regression.md b/docs/guide/visual-regression.md new file mode 100644 index 0000000..8c72067 --- /dev/null +++ b/docs/guide/visual-regression.md @@ -0,0 +1,233 @@ +# Visual Regression + +`toHaveScreenshot()` compares a screenshot of a page or an element with a baseline image stored next to your tests. +It catches what the other assertions cannot see: a broken layout, a wrong color, a font that stopped loading. + +```php +public function testCardLooksTheSame(): void +{ + $this->page->goto('/design-system/card'); + + $this->expect($this->page->getByTestId('preview'))->toHaveScreenshot('card.webp'); +} +``` + +## Why compare pixels + +Every other assertion reads the DOM: `toBeVisible()`, `toHaveText()` and `toHaveCSS()` check one element and one property +at a time. What the user sees is the rendered result of all of them together, and the two can disagree. A page where +every element is attached, visible and correctly labelled can still look broken: only the rendered pixels tell. + +A screenshot comparison checks that rendered result as a whole. It catches the bugs no DOM assertion is written for: + +- **CSS side effects.** A margin changed in a shared stylesheet, or a design token renamed, and a card three pages away + overflows its container. +- **Design system drift.** A button variant loses its border radius, the dark theme picks the wrong background, an icon + moves by 4 pixels. +- **Assets and fonts.** A web font fails to load and the fallback reflows the page, an SVG icon goes missing, an image + is stretched. +- **Layout at breakpoints.** Text wraps differently, elements overlap, a sticky header covers content on a narrow + viewport. +- **Upgrades.** A new version of a CSS framework, a UI component library or the browser itself, with every visual + consequence visible in one run. + +Describing "this looks right" with DOM assertions would take hundreds of `toHaveCSS()` checks per component, and they +would still miss how elements interact. One screenshot covers it, and the diff image points at the exact place that +changed: cheap to write, fast to review. + +### A defensive test, on chosen surfaces + +Visual regression is a guard against unwanted change. It belongs on the few surfaces whose appearance is part of the +contract: the design system, the pages that carry the product. Every baseline is a file to review each time the +design moves on purpose, so screenshotting every page of an application turns each redesign into a chore and each +failure into noise. A handful of well-chosen baselines stays meaningful: when one fails, something that mattered +changed. + +### Where it pays off + +- **A component gallery.** One page renders each component in every variant, one screenshot per component and theme. + A data provider turns it into a matrix: + + ```php + #[DataProvider('components')] + public function testComponentLooksTheSame(string $name, string $theme): void + { + $this->page->emulateMedia(['colorScheme' => $theme]); + $this->page->goto('/design-system/'.$name); + + $this->expect($this->page->getByTestId('preview'))->toHaveScreenshot(sprintf('%s-%s.webp', $name, $theme)); + } + + public static function components(): iterable + { + foreach (['button', 'badge', 'alert', 'card', 'modal'] as $name) { + foreach (['light', 'dark'] as $theme) { + yield sprintf('%s %s', $name, $theme) => [$name, $theme]; + } + } + } + ``` + +- **Precious pages.** Login, checkout, landing pages: the screens where a visual break costs money or trust. +- **Refactors and upgrades.** Record the baselines, change the code, and see exactly what moved. + +### What it complements + +A screenshot checks appearance. Clicking, submitting and navigating stay the job of behavior assertions, which prove +the application works; visual regression proves it still looks the way it was designed. Pages full of dynamic content +(dates, avatars, ads, user data) need masks or fixed fixtures to compare reliably, and baselines need one reference +system, since fonts render differently from one system to another. Both are covered below. + +## How it works + +The comparison runs in Playwright itself, with the same code as `toHaveScreenshot()` in `@playwright/test`: + +1. Playwright takes screenshots until two consecutive ones are identical. Animations are disabled, the text caret is + hidden and the image uses CSS pixels, so the page has a chance to settle. +2. The stable screenshot is compared with the baseline, pixel by pixel, within the tolerance you configure. +3. When they differ, the expected, actual and diff images are written to the test's own directory under + `test-failures/`, and the assertion fails with their paths. + +## Baselines + +The first run has nothing to compare with: it writes the baseline and fails, so a new screenshot is always reviewed +before it is trusted. Run the test again and it passes. + +A relative name is stored next to the test file, suffixed with the browser and the platform, like `@playwright/test` +does. Both are read from the page under test, nothing to configure. Browsers and systems render fonts differently, so +each combination keeps its own baseline: + +``` +tests/E2E/CardTest.php +tests/E2E/CardTest.php-snapshots/card-chromium-darwin.webp +tests/E2E/CardTest.php-snapshots/card-chromium-linux.webp +tests/E2E/CardTest.php-snapshots/card-firefox-linux.webp +``` + +A persistent context has no browser object: its baselines carry the platform only. An absolute path is used as is. To +store baselines elsewhere, override `snapshotDirectory()` in your test case. + +Commit the baselines. Generate the ones used in CI on the same system as CI, ideally in the same Docker image. + +Record and compare in headless mode, the default and what CI runs. Both headless modes of Chromium, the headless shell +and `channel: 'chromium'`, produce byte-identical screenshots. A visible browser shows a scrollbar as soon as the page +scrolls, 15 pixels that fail any full-page comparison against a headless baseline. + +### Failure images + +Each test writes its failure images in its own directory, one per browser, as Playwright JS does per test and project +under `test-results/`: `test-failures/--/`. Override `testOutputDirectory()` in your +test case to move it. The images are named after the name given to `toHaveScreenshot()`, the directory already telling +the browser. For `toHaveScreenshot('card.png')` in `CardTest::testCard`, run in Chromium, the baseline is +`card-chromium-linux.png` and a failure writes to `test-failures/CardTest-testCard-chromium/`: + +| File | Content | +|---|---| +| `card-expected.png` | Copy of the baseline at the time of the failure | +| `card-actual.png` | Last screenshot taken | +| `card-diff.png` | Baseline in pale grey, differing pixels in red. Always a PNG | + +Expected and actual keep the format of the baseline, `.webp` included. The three files are removed as soon as the +baseline matches again or is recorded again, and the directory with them once empty: a green run leaves a clean +`test-failures/`. + +What happens to the baseline itself: + +| Situation | `missing` (default) | `none` | `changed` | `all` | +|---|---|---|---|---| +| No baseline | written, test fails | test fails | written, test passes | written, test passes | +| Baseline matches | kept, test passes | kept, test passes | kept, test passes | rewritten, test passes | +| Baseline differs | kept, failure images, test fails | kept, failure images, test fails | rewritten, test passes | rewritten, test passes | +| Page never stable, baseline present | kept, failure images, test fails | kept, failure images, test fails | kept, failure images, test fails | kept, test fails | +| Page never stable, no baseline | nothing written, test fails | nothing written, test fails | nothing written, test fails | nothing written, test fails | + +### Prefer WebP + +The extension of the name picks the format. Without extension, `.png` is added, as in `@playwright/test`. We recommend +`.webp`: Playwright records it lossless, pixel for pixel the same image as the PNG. + +```php +$this->expect($this->page)->toHaveScreenshot('dashboard.webp'); +``` + +| | PNG | WebP | +|---|---|---| +| Size, flat interface | 12 KB | 3 KB | +| Size, viewport with gradients | 357 KB | 24 KB | +| Comparison, viewport | 88 ms | 21 ms | +| Comparison, full page | 423 ms | 153 ms | + +A repository that stores hundreds of baselines stays light, and the suite runs faster since decoding dominates the +comparison. PNG keeps one advantage: more review tools preview it in a diff. The diff image of a failure is always a PNG. +Other extensions, JPEG included, throw an `InvalidArgumentException`: only lossless formats compare reliably. + +### Updating baselines + +The `PLAYWRIGHT_UPDATE_SNAPSHOTS` variable takes the values of the `--update-snapshots` flag of `@playwright/test`: + +| Value | Missing baseline | Different screenshot | +|---|---|---| +| `missing` (default) | written, the test fails | the test fails | +| `none` | the test fails | the test fails | +| `changed` | written, the test passes | rewritten, the test passes | +| `all` | written, the test passes | rewritten, matching ones too | + +```shell +PLAYWRIGHT_UPDATE_SNAPSHOTS=changed vendor/bin/phpunit tests/E2E +``` + +Then review the changed images in your diff before committing them. + +## Options + +Pass a `ToHaveScreenshotOptions` with named arguments to tune the comparison: + +```php +use Playwright\Assertions\Options\ToHaveScreenshotOptions; + +$this->expect($this->page)->toHaveScreenshot('dashboard.webp', new ToHaveScreenshotOptions( + fullPage: true, + maxDiffPixelRatio: 0.01, + mask: [$this->page->locator('.timestamp'), $this->page->locator('.avatar')], +)); +``` + +The defaults are the ones of `@playwright/test`: + +| Option | Default | Effect | +|---|---|---| +| `threshold` | `0.2` | Color distance tolerated for each pixel, from 0 (exact) to 1. Used by pixelmatch only | +| `maxDiffPixels` | none | Number of differing pixels tolerated | +| `maxDiffPixelRatio` | none | Share of differing pixels tolerated, from 0 to 1 | +| `comparator` | `'pixelmatch'` | `'ssim-cie94'` compares structure and perceived color, experimental in Playwright | +| `mask` | `[]` | Locators covered with a solid box, for dates, avatars, ads. Inside an iframe, use `frameLocator()` | +| `maskColor` | `'#FF00FF'` | Any CSS color | +| `style` | none | CSS applied while taking the screenshot, to hide or freeze dynamic parts | +| `fullPage` | `false` | Captures the whole scrollable page. Pages only | +| `clip` | none | Region to capture: `['x' => 0, 'y' => 0, 'width' => 320, 'height' => 200]`. Pages only | +| `omitBackground` | `false` | Transparent background instead of white | +| `animations` | `'disabled'` | Finite animations jump to their end, infinite ones are cancelled. `'allow'` keeps them running | +| `caret` | `'hide'` | `'initial'` keeps the blinking text caret | +| `scale` | `'css'` | One pixel per CSS pixel, identical on every screen. `'device'` is sharper on high-DPI screens | +| `timeoutMs` | assertion timeout | Time allowed to get two identical consecutive screenshots | +| `message` | generated | Replaces the first line of the failure message | + +Invalid values throw an `InvalidArgumentException` when the options are created, so a typo such as `comparator: 'ssim'` +fails in PHP with the list of accepted values. On a locator, `fullPage` and `clip` throw too: an element is captured whole. + +### Tolerating differences + +Without tolerance, a single differing pixel fails the assertion. `maxDiffPixels` suits a fixed-size element, +`maxDiffPixelRatio` a page whose size varies. When both are set, the stricter one applies. + +### Choosing a threshold + +The default `threshold` of `0.2` forgives anti-aliasing and small rendering noise. It also forgives close shades of one +hue: `#818cf8` and `#6366f1`, two indigos of the same palette, compare as identical. To catch a design token change, lower +it to `0.1` or switch to the `ssim-cie94` comparator, which ignores `threshold`. + +## Limits + +- Negation throws a `LogicException`: a screenshot either matches its baseline or is reviewed. +- Decorated pages and test doubles throw a `LogicException` too. The comparison needs the `Page` and `Locator` created + by Playwright PHP. From fde0be3483adf9ad6bd05e5d781f8c7d6fd4ca28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Andr=C3=A9?= Date: Sun, 27 Sep 2026 00:06:43 +0200 Subject: [PATCH 4/5] Isolate screenshot artifacts and reject non-finite options --- docs/guide/visual-regression.md | 11 +- .../Internal/ScreenshotExpectation.php | 13 ++- .../Options/ToHaveScreenshotOptions.php | 8 ++ .../Screenshot/ToHaveScreenshotTest.php | 41 ++++++- .../Internal/ScreenshotExpectationTest.php | 107 +++++++++++++++--- .../Unit/Assertions/LocatorAssertionsTest.php | 12 +- .../Options/ToHaveScreenshotOptionsTest.php | 12 ++ tests/Unit/Assertions/PageAssertionsTest.php | 7 +- tests/Unit/Testing/ExpectFactoryTest.php | 7 +- 9 files changed, 183 insertions(+), 35 deletions(-) diff --git a/docs/guide/visual-regression.md b/docs/guide/visual-regression.md index 8c72067..90920bd 100644 --- a/docs/guide/visual-regression.md +++ b/docs/guide/visual-regression.md @@ -117,9 +117,11 @@ scrolls, 15 pixels that fail any full-page comparison against a headless baselin Each test writes its failure images in its own directory, one per browser, as Playwright JS does per test and project under `test-results/`: `test-failures/--/`. Override `testOutputDirectory()` in your -test case to move it. The images are named after the name given to `toHaveScreenshot()`, the directory already telling -the browser. For `toHaveScreenshot('card.png')` in `CardTest::testCard`, run in Chromium, the baseline is -`card-chromium-linux.png` and a failure writes to `test-failures/CardTest-testCard-chromium/`: +test case to move it. Each baseline has a subdirectory with its name and a short identifier derived from its path. +This keeps `light/card.png`, `dark/card.png` and `card.webp` separate, including when a passing assertion cleans up +its earlier failure. The image names omit the browser, which the test directory already identifies. +For `toHaveScreenshot('card.png')` in `CardTest::testCard`, run in Chromium, the baseline is +`card-chromium-linux.png` and a failure writes to `test-failures/CardTest-testCard-chromium/card-/`: | File | Content | |---|---| @@ -213,7 +215,8 @@ The defaults are the ones of `@playwright/test`: | `message` | generated | Replaces the first line of the failure message | Invalid values throw an `InvalidArgumentException` when the options are created, so a typo such as `comparator: 'ssim'` -fails in PHP with the list of accepted values. On a locator, `fullPage` and `clip` throw too: an element is captured whole. +fails in PHP with the list of accepted values. Numeric options must be finite: `NAN` and infinities are rejected before +reaching the browser. On a locator, `fullPage` and `clip` throw too: an element is captured whole. ### Tolerating differences diff --git a/src/Assertions/Internal/ScreenshotExpectation.php b/src/Assertions/Internal/ScreenshotExpectation.php index 7bedb80..f3c9a70 100644 --- a/src/Assertions/Internal/ScreenshotExpectation.php +++ b/src/Assertions/Internal/ScreenshotExpectation.php @@ -118,8 +118,10 @@ private function removeArtifacts(string $path): void } } - if (is_dir($this->outputDir()) && [] === array_diff(scandir($this->outputDir()) ?: [], ['.', '..'])) { - rmdir($this->outputDir()); + foreach ([dirname($stem), $this->outputDir()] as $directory) { + if (is_dir($directory) && [] === array_diff(scandir($directory) ?: [], ['.', '..'])) { + rmdir($directory); + } } } @@ -195,7 +197,8 @@ private function updateMode(): string /** * Failure images drop the variant of the baseline name, which their * directory already carries: "card-chromium-linux.png" fails as - * "card-actual.png", as in Playwright JS. + * "card-actual.png", as in Playwright JS. Each baseline gets its own + * directory so matching one cannot erase another baseline's failure. */ private function artifactStem(string $path): string { @@ -204,7 +207,9 @@ private function artifactStem(string $path): string $stem = substr($stem, 0, -\strlen($this->variant) - 1); } - return $this->outputDir().'/'.$stem; + $identity = substr(hash('sha256', realpath($path) ?: $path), 0, 12); + + return $this->outputDir().'/'.$stem.'-'.$identity.'/'.$stem; } private function outputDir(): string diff --git a/src/Assertions/Options/ToHaveScreenshotOptions.php b/src/Assertions/Options/ToHaveScreenshotOptions.php index 75c8557..29baea0 100644 --- a/src/Assertions/Options/ToHaveScreenshotOptions.php +++ b/src/Assertions/Options/ToHaveScreenshotOptions.php @@ -152,6 +152,10 @@ public function toArray(): array private static function assertRange(string $option, float $value): void { + if (!is_finite($value)) { + throw new \InvalidArgumentException(sprintf('The %s option must be between 0 and 1, a non-finite value given.', $option)); + } + if ($value < 0 || $value > 1) { throw new \InvalidArgumentException(sprintf('The %s option must be between 0 and 1, %s given.', $option, $value)); } @@ -176,6 +180,10 @@ private static function assertClip(array $clip): void if (!isset($clip[$key]) || !is_int($clip[$key]) && !is_float($clip[$key])) { throw new \InvalidArgumentException(sprintf('The clip option needs numeric "x", "y", "width" and "height" keys, "%s" is missing or not a number.', $key)); } + + if (!is_finite($clip[$key])) { + throw new \InvalidArgumentException(sprintf('The clip option needs a finite "%s" value.', $key)); + } } if ($clip['width'] <= 0 || $clip['height'] <= 0) { diff --git a/tests/Functional/Screenshot/ToHaveScreenshotTest.php b/tests/Functional/Screenshot/ToHaveScreenshotTest.php index 9c78968..84ac42a 100644 --- a/tests/Functional/Screenshot/ToHaveScreenshotTest.php +++ b/tests/Functional/Screenshot/ToHaveScreenshotTest.php @@ -108,10 +108,11 @@ public function testChangedElementFailsAndKeepsTheImagesUntilItPassesAgain(): vo } catch (AssertionException $e) { $this->assertStringContainsString('Screenshot of the locator does not match', $e->getMessage()); $this->assertStringContainsString('pixels', $e->getMessage()); + $this->assertNotNull($e->actual); + $out = dirname($e->actual).'/card'; } - $out = $this->testOutputDirectory().'-chromium/card'; - $this->assertStringEndsWith('/test-failures/ToHaveScreenshotTest-testChangedElementFailsAndKeepsTheImagesUntilItPassesAgain-chromium/card', $out); + $this->assertStringStartsWith($this->testOutputDirectory().'-chromium/', $out); $this->assertFileExists($out.'-expected.png'); $this->assertFileExists($out.'-actual.png'); $this->assertFileExists($out.'-diff.png'); @@ -124,6 +125,36 @@ public function testChangedElementFailsAndKeepsTheImagesUntilItPassesAgain(): vo $this->assertFileDoesNotExist($out.'-diff.png'); } + public function testMatchingAnotherBaselineKeepsTheFailureImages(): void + { + $this->card('#818cf8', 'Light'); + $card = $this->page->locator('#card'); + $options = new ToHaveScreenshotOptions(mask: [$this->page->locator('.time')]); + $this->record(fn () => $this->expect($card)->toHaveScreenshot('light/card.png', $options)); + + $this->card('#f472b6', 'Dark'); + $this->record(fn () => $this->expect($card)->toHaveScreenshot('dark/card.png', $options)); + + try { + $this->expect($card)->toHaveScreenshot('light/card.png', $options); + $this->fail('A changed element must fail the assertion.'); + } catch (AssertionException $e) { + $this->assertNotNull($e->actual); + $outputDir = dirname($e->actual); + } + + $this->expect($card)->toHaveScreenshot('dark/card.png', $options); + + foreach (['expected', 'actual', 'diff'] as $kind) { + $this->assertFileExists($outputDir.'/card-'.$kind.'.png'); + } + + $this->card('#818cf8', 'Light'); + $this->expect($card)->toHaveScreenshot('light/card.png', $options); + + $this->assertDirectoryDoesNotExist($outputDir); + } + public function testDefaultThresholdToleratesCloseShadesOfOneHue(): void { $this->card('#818cf8'); @@ -185,10 +216,12 @@ public function testWebpBaselineIsLosslessAndComparedAsWebp(): void try { $this->expect($card)->toHaveScreenshot('card.webp', $options); $this->fail('A changed element must fail the assertion.'); - } catch (AssertionException) { + } catch (AssertionException $e) { + $this->assertNotNull($e->actual); + $out = dirname($e->actual).'/card'; } - $out = $this->testOutputDirectory().'-chromium/card'; + $this->assertStringStartsWith($this->testOutputDirectory().'-chromium/', $out); $this->assertSame('WEBP', substr((string) file_get_contents($out.'-actual.webp'), 8, 4)); $this->assertSame("\x89PNG", substr((string) file_get_contents($out.'-diff.png'), 0, 4)); } diff --git a/tests/Unit/Assertions/Internal/ScreenshotExpectationTest.php b/tests/Unit/Assertions/Internal/ScreenshotExpectationTest.php index 9a9864e..445087c 100644 --- a/tests/Unit/Assertions/Internal/ScreenshotExpectationTest.php +++ b/tests/Unit/Assertions/Internal/ScreenshotExpectationTest.php @@ -145,12 +145,12 @@ public function testMismatchKeepsExpectedActualAndDiff(): void $this->assertStringContainsString("Screenshot of the page does not match {$path}.", $e->getMessage()); $this->assertStringContainsString('120 pixels (ratio 0.02 of all image pixels) are different.', $e->getMessage()); $this->assertStringContainsString('PLAYWRIGHT_UPDATE_SNAPSHOTS=changed', $e->getMessage()); - $this->assertSame($this->dir.'/out/card-actual.png', $e->actual); + $this->assertSame($this->artifact('card-actual.png'), $e->actual); } - $this->assertSame('baseline', file_get_contents($this->dir.'/out/card-expected.png')); - $this->assertSame('actual', file_get_contents($this->dir.'/out/card-actual.png')); - $this->assertSame('diff', file_get_contents($this->dir.'/out/card-diff.png')); + $this->assertSame('baseline', file_get_contents($this->artifact('card-expected.png'))); + $this->assertSame('actual', file_get_contents($this->artifact('card-actual.png'))); + $this->assertSame('diff', file_get_contents($this->artifact('card-diff.png'))); $this->assertSame('baseline', file_get_contents($path)); } @@ -190,6 +190,73 @@ public function testFailureKeepsTheImagesOfOtherBaselines(): void $this->assertFileExists($this->dir.'/out/other-actual.png'); } + #[DataProvider('provideCollidingBaselineNames')] + public function testArtifactsBelongOnlyToTheirBaseline(string $firstName, string $secondName, string $cleanupMode): void + { + $paths = [$this->dir.'/'.$firstName, $this->dir.'/'.$secondName]; + $directories = []; + + foreach ($paths as $index => $path) { + if (!is_dir(dirname($path))) { + mkdir(dirname($path), 0777, true); + } + file_put_contents($path, 'baseline '.$index); + + try { + $this->expectation(new ScreenshotComparison(actual: 'actual '.$index, diff: 'diff '.$index, errorMessage: 'different')) + ->assert($path, new ToHaveScreenshotOptions(), 5000); + $this->fail('A mismatch must fail the assertion.'); + } catch (AssertionException $e) { + $this->assertNotNull($e->actual); + $directories[] = dirname($e->actual); + } + } + + $this->assertNotSame($directories[0], $directories[1]); + foreach ($paths as $index => $path) { + $extension = pathinfo($path, \PATHINFO_EXTENSION); + $this->assertSame('baseline '.$index, file_get_contents($directories[$index].'/card-expected.'.$extension)); + $this->assertSame('actual '.$index, file_get_contents($directories[$index].'/card-actual.'.$extension)); + $this->assertSame('diff '.$index, file_get_contents($directories[$index].'/card-diff.png')); + } + + $result = match ($cleanupMode) { + 'changed' => new ScreenshotComparison(actual: 'updated', errorMessage: 'different'), + 'all' => new ScreenshotComparison(actual: 'updated'), + default => new ScreenshotComparison(actual: null), + }; + $this->expectation($result, $cleanupMode)->assert($paths[0], new ToHaveScreenshotOptions(), 5000); + + $this->assertDirectoryDoesNotExist($directories[0]); + $this->assertSame('diff 1', file_get_contents($directories[1].'/card-diff.png')); + + $this->expectation(new ScreenshotComparison(actual: null))->assert($paths[1], new ToHaveScreenshotOptions(), 5000); + + $this->assertDirectoryDoesNotExist($this->dir.'/out'); + } + + /** + * @return iterable + */ + public static function provideCollidingBaselineNames(): iterable + { + foreach (['none', 'changed', 'all'] as $mode) { + yield 'different directories, '.$mode => ['a/card.png', 'b/card.png', $mode]; + yield 'different formats, '.$mode => ['card.png', 'card.webp', $mode]; + } + } + + public function testMatchingTheSameBaselineThroughAnotherPathCleansItsArtifacts(): void + { + $this->baseline('baseline'); + $this->staleArtifacts(); + + $this->expectation(new ScreenshotComparison(actual: null)) + ->assert($this->dir.'/./card.png', new ToHaveScreenshotOptions(), 5000); + + $this->assertDirectoryDoesNotExist($this->dir.'/out'); + } + public function testFailureImagesDropTheVariantOfTheBaselineName(): void { $path = $this->dir.'/card-chromium-linux.webp'; @@ -202,7 +269,7 @@ public function testFailureImagesDropTheVariantOfTheBaselineName(): void } catch (AssertionException) { } - $this->assertSame(['card-actual.webp', 'card-diff.png', 'card-expected.webp'], array_map('basename', glob($this->dir.'/out/*') ?: [])); + $this->assertSame(['card-actual.webp', 'card-diff.png', 'card-expected.webp'], array_map('basename', glob($this->dir.'/out/*/*') ?: [])); $this->expectation(new ScreenshotComparison(actual: null), variant: 'chromium-linux')->assert($path, new ToHaveScreenshotOptions(), 5000); @@ -219,7 +286,7 @@ public function testFailureImagesKeepANameThatDoesNotEndWithTheVariant(): void } catch (AssertionException) { } - $this->assertFileExists($this->dir.'/out/card-actual.png'); + $this->assertFileExists($this->artifact('card-actual.png')); } public function testMismatchUsesTheCustomMessage(): void @@ -315,9 +382,9 @@ public function testWebpBaselineRequestsWebpAndKeepsWebpArtifacts(): void } $this->assertSame('webp', $this->calls[0]['options']['type']); - $this->assertFileExists($this->dir.'/out/card-expected.webp'); - $this->assertFileExists($this->dir.'/out/card-actual.webp'); - $this->assertFileExists($this->dir.'/out/card-diff.png'); + $this->assertFileExists($this->artifact('card-expected.webp')); + $this->assertFileExists($this->artifact('card-actual.webp')); + $this->assertFileExists($this->artifact('card-diff.png')); } public function testJpegBaselineIsRejected(): void @@ -341,8 +408,8 @@ public function testUnstablePageWithABaselineKeepsTheLastScreenshot(): void } $this->assertSame('baseline', file_get_contents($path)); - $this->assertSame('baseline', file_get_contents($this->dir.'/out/card-expected.png')); - $this->assertSame('moving', file_get_contents($this->dir.'/out/card-actual.png')); + $this->assertSame('baseline', file_get_contents($this->artifact('card-expected.png'))); + $this->assertSame('moving', file_get_contents($this->artifact('card-actual.png'))); } public function testUnreadableBaselineIsReported(): void @@ -377,7 +444,7 @@ public function testUncreatableOutputDirectoryIsReported(): void ); $this->expectException(\RuntimeException::class); - $this->expectExceptionMessage(sprintf('Directory "%s/file/out" was not created.', $this->dir)); + $this->expectExceptionMessage(sprintf('Directory "%s/file/out/', $this->dir)); $expectation->assert($path, new ToHaveScreenshotOptions(), 5000); } @@ -401,11 +468,21 @@ public function testUnknownUpdateModeIsRejected(): void $this->expectation(new ScreenshotComparison(actual: 'stable'), 'yes')->assert($this->dir.'/card.png', new ToHaveScreenshotOptions(), 5000); } + private function artifact(string $name): string + { + $files = glob($this->dir.'/out/*/'.$name) ?: []; + $this->assertCount(1, $files); + + return $files[0]; + } + private function staleArtifacts(): void { - mkdir($this->dir.'/out'); - foreach (['card-expected.png', 'card-actual.png', 'card-diff.png'] as $file) { - file_put_contents($this->dir.'/out/'.$file, 'stale'); + try { + $this->expectation(new ScreenshotComparison(actual: 'stale', diff: 'stale', errorMessage: 'different')) + ->assert($this->dir.'/card.png', new ToHaveScreenshotOptions(), 5000); + $this->fail('A mismatch must fail the assertion.'); + } catch (AssertionException) { } } diff --git a/tests/Unit/Assertions/LocatorAssertionsTest.php b/tests/Unit/Assertions/LocatorAssertionsTest.php index d660f22..7acac2c 100644 --- a/tests/Unit/Assertions/LocatorAssertionsTest.php +++ b/tests/Unit/Assertions/LocatorAssertionsTest.php @@ -442,14 +442,18 @@ public function testToHaveScreenshotWritesFailureImagesInTheGivenDirectory(): vo try { (new LocatorAssertions(new Locator($transport, 'page1', '#card'), null, $dir.'/out'))->toHaveScreenshot($path); $this->fail('A mismatch must fail the assertion.'); - } catch (AssertionException) { + } catch (AssertionException $e) { + $this->assertNotNull($e->actual); + $outputDir = dirname($e->actual); } - $this->assertSame('actual', file_get_contents($dir.'/out/card-actual.png')); - $this->assertSame('diff', file_get_contents($dir.'/out/card-diff.png')); - foreach (glob($dir.'/out/*') ?: [] as $file) { + $this->assertStringStartsWith($dir.'/out/', $outputDir); + $this->assertSame('actual', file_get_contents($outputDir.'/card-actual.png')); + $this->assertSame('diff', file_get_contents($outputDir.'/card-diff.png')); + foreach (glob($outputDir.'/*') ?: [] as $file) { unlink($file); } + rmdir($outputDir); rmdir($dir.'/out'); } diff --git a/tests/Unit/Assertions/Options/ToHaveScreenshotOptionsTest.php b/tests/Unit/Assertions/Options/ToHaveScreenshotOptionsTest.php index bf9e665..e1b50a0 100644 --- a/tests/Unit/Assertions/Options/ToHaveScreenshotOptionsTest.php +++ b/tests/Unit/Assertions/Options/ToHaveScreenshotOptionsTest.php @@ -126,6 +126,18 @@ public function testRejectsInvalidOptions(array $arguments, string $message): vo */ public static function provideInvalidOptions(): iterable { + foreach (['NaN' => \NAN, 'positive infinity' => \INF, 'negative infinity' => -\INF] as $label => $value) { + foreach (['threshold', 'maxDiffPixelRatio'] as $option) { + yield $option.' '.$label => [[$option => $value], sprintf('The %s option must be between 0 and 1', $option)]; + } + foreach (['x', 'y', 'width', 'height'] as $coordinate) { + $clip = ['x' => 0, 'y' => 0, 'width' => 10, 'height' => 10]; + $clip[$coordinate] = $value; + + yield 'clip '.$coordinate.' '.$label => [['clip' => $clip], sprintf('The clip option needs a finite "%s" value.', $coordinate)]; + } + } + yield 'threshold above 1' => [['threshold' => 2.0], 'The threshold option must be between 0 and 1, 2 given.']; yield 'negative threshold' => [['threshold' => -0.1], 'The threshold option must be between 0 and 1, -0.1 given.']; yield 'ratio above 1' => [['maxDiffPixelRatio' => 1.5], 'The maxDiffPixelRatio option must be between 0 and 1, 1.5 given.']; diff --git a/tests/Unit/Assertions/PageAssertionsTest.php b/tests/Unit/Assertions/PageAssertionsTest.php index 9fc307e..f09e5e5 100644 --- a/tests/Unit/Assertions/PageAssertionsTest.php +++ b/tests/Unit/Assertions/PageAssertionsTest.php @@ -191,8 +191,11 @@ public function testToHaveScreenshotClosesTheTracingGroupOnFailure(): void try { (new PageAssertions($page, $tracing))->toHaveScreenshot($path); } finally { - foreach (glob(getcwd().'/test-failures/snapshots/page-*') ?: [] as $file) { - unlink($file); + foreach (glob(getcwd().'/test-failures/snapshots/page-*', GLOB_ONLYDIR) ?: [] as $directory) { + foreach (glob($directory.'/*') ?: [] as $file) { + unlink($file); + } + rmdir($directory); } @rmdir(getcwd().'/test-failures/snapshots'); } diff --git a/tests/Unit/Testing/ExpectFactoryTest.php b/tests/Unit/Testing/ExpectFactoryTest.php index 8e93db1..eb6b00a 100644 --- a/tests/Unit/Testing/ExpectFactoryTest.php +++ b/tests/Unit/Testing/ExpectFactoryTest.php @@ -249,15 +249,18 @@ public function testFailureImagesGoToADirectoryPerBrowser(): void try { (new Expect($page, null, $dir, $dir.'/CardTest-testCard'))->toHaveScreenshot('card.png'); $this->fail('A mismatch must fail the assertion.'); - } catch (AssertionException) { + } catch (AssertionException $e) { + $this->assertNotNull($e->actual); + $out = dirname($e->actual); } - $out = $dir.'/CardTest-testCard-firefox'; + $this->assertStringStartsWith($dir.'/CardTest-testCard-firefox/', $out); $this->assertSame(['card-actual.png', 'card-diff.png', 'card-expected.png'], array_map('basename', glob($out.'/*') ?: [])); foreach (glob($out.'/*') ?: [] as $file) { unlink($file); } rmdir($out); + rmdir(dirname($out)); } public function testALocatorWithoutPageKeepsThePlatformOnly(): void From ea7f5c0cd544d1d9289fbebe7534693433b5a00f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Andr=C3=A9?= Date: Sun, 27 Sep 2026 13:25:47 +0000 Subject: [PATCH 5/5] Address review of the visual regression assertion - An empty baseline reached Playwright as "no baseline" and matched any screenshot. It now fails with a clear message, and is recorded again in the changed and all modes. The bridge sends any string it receives. - A failure removes the images of the previous one, so a diff from an earlier mismatch never sits next to a later timeout. - Locator::normalize() keeps the page of the locator, so a normalized locator names its baselines after the browser too. - UNC paths (\\server\share\card.png) are absolute snapshot paths. - The unreadable baseline test skips without POSIX permissions. - The useItemsLocator() docblock is back on its helper. --- bin/lib/handlers.js | 2 +- .../Internal/ScreenshotExpectation.php | 11 +++- src/Locator/Locator.php | 2 +- src/Testing/Expect.php | 2 +- .../Screenshot/ToHaveScreenshotTest.php | 12 ++++ .../Internal/ScreenshotExpectationTest.php | 58 +++++++++++++++++++ tests/Unit/Locator/LocatorTest.php | 17 ++++-- tests/Unit/Testing/ExpectFactoryTest.php | 10 ++++ 8 files changed, 106 insertions(+), 8 deletions(-) diff --git a/bin/lib/handlers.js b/bin/lib/handlers.js index 071449a..a2ed8f0 100644 --- a/bin/lib/handlers.js +++ b/bin/lib/handlers.js @@ -21,7 +21,7 @@ async function expectScreenshot(page, locator, command) { ...options, mask, locator: locator || undefined, - expected: command.expected ? Buffer.from(command.expected, 'base64') : undefined, + expected: typeof command.expected === 'string' ? Buffer.from(command.expected, 'base64') : undefined, isNot: false, }); diff --git a/src/Assertions/Internal/ScreenshotExpectation.php b/src/Assertions/Internal/ScreenshotExpectation.php index f3c9a70..30a4b62 100644 --- a/src/Assertions/Internal/ScreenshotExpectation.php +++ b/src/Assertions/Internal/ScreenshotExpectation.php @@ -57,9 +57,16 @@ public function assert(string $path, ToHaveScreenshotOptions $options, int $time throw new \RuntimeException(sprintf('Unable to read the snapshot "%s".', $path)); } + // An empty file would reach Playwright as "no baseline" and match + // anything: it only passes once recorded again. + if ('' === $expected && 'changed' !== $mode && 'all' !== $mode) { + throw new AssertionException($options->message ?? sprintf('The snapshot at %s is empty. Run with %s=changed to record it again.', $path, self::UPDATE_ENV)); + } + // "all" records every screenshot, like --update-snapshots=all: no // comparison, Playwright returns no image when a comparison matches. - if (null === $expected || 'all' === $mode) { + // "changed" records an empty baseline the same way. + if (null === $expected || '' === $expected || 'all' === $mode) { $this->record($path, $mode, null === $expected, ($this->compare)(null, $payload), $options); return; @@ -130,6 +137,8 @@ private function removeArtifacts(string $path): void */ private function writeArtifacts(string $path, string $expected, ScreenshotComparison $result): array { + $this->removeArtifacts($path); + $stem = $this->artifactStem($path); $extension = '.'.$this->imageType($path); diff --git a/src/Locator/Locator.php b/src/Locator/Locator.php index 88b7ce6..a3effc3 100644 --- a/src/Locator/Locator.php +++ b/src/Locator/Locator.php @@ -1022,7 +1022,7 @@ public function normalize(): self throw new ProtocolErrorException('Invalid normalize response', 0); } - return new self($this->transport, $this->pageId, $value, $this->frameSelector, $this->logger); + return new self($this->transport, $this->pageId, $value, $this->frameSelector, $this->logger, [], $this->page); } public function contentFrame(): FrameLocatorInterface diff --git a/src/Testing/Expect.php b/src/Testing/Expect.php index f529fe9..1ab1805 100644 --- a/src/Testing/Expect.php +++ b/src/Testing/Expect.php @@ -188,7 +188,7 @@ public function toHaveScreenshot(string $name, ?ToHaveScreenshotOptions $options private function snapshotPath(string $name): string { - if (str_starts_with($name, '/') || 1 === preg_match('~^[a-zA-Z]:[\\\\/]~', $name)) { + if (str_starts_with($name, '/') || str_starts_with($name, '\\\\') || 1 === preg_match('~^[a-zA-Z]:[\\\\/]~', $name)) { return $name; } diff --git a/tests/Functional/Screenshot/ToHaveScreenshotTest.php b/tests/Functional/Screenshot/ToHaveScreenshotTest.php index 84ac42a..4041673 100644 --- a/tests/Functional/Screenshot/ToHaveScreenshotTest.php +++ b/tests/Functional/Screenshot/ToHaveScreenshotTest.php @@ -304,6 +304,18 @@ public function testNoneModeNeverWritesABaseline(): void $this->assertFileDoesNotExist($this->dir.'/card-'.$this->variant().'.png'); } + public function testEmptyBaselineFailsThroughTheBridge(): void + { + $this->card('#818cf8'); + mkdir($this->dir, 0777, true); + file_put_contents($this->dir.'/card-'.$this->variant().'.png', ''); + + $this->expectException(AssertionException::class); + $this->expectExceptionMessage('is empty'); + + $this->expect($this->page->locator('#card'))->toHaveScreenshot('card.png'); + } + public function testFullPageIsRejectedOnAnElement(): void { $this->card('#818cf8'); diff --git a/tests/Unit/Assertions/Internal/ScreenshotExpectationTest.php b/tests/Unit/Assertions/Internal/ScreenshotExpectationTest.php index 445087c..5410b65 100644 --- a/tests/Unit/Assertions/Internal/ScreenshotExpectationTest.php +++ b/tests/Unit/Assertions/Internal/ScreenshotExpectationTest.php @@ -414,6 +414,9 @@ public function testUnstablePageWithABaselineKeepsTheLastScreenshot(): void public function testUnreadableBaselineIsReported(): void { + if ('Windows' === \PHP_OS_FAMILY || !function_exists('posix_geteuid')) { + $this->markTestSkipped('Needs POSIX file permissions.'); + } if (0 === posix_geteuid()) { $this->markTestSkipped('Root reads files whatever their permissions.'); } @@ -460,6 +463,61 @@ public function testUnwritableBaselineIsReported(): void $this->expectation(new ScreenshotComparison(actual: 'stable'), 'changed')->assert($path, new ToHaveScreenshotOptions(), 5000); } + /** + * @param 'none'|'missing' $mode + */ + #[DataProvider('provideComparingModes')] + public function testEmptyBaselineFailsWithoutComparing(string $mode): void + { + $path = $this->baseline(''); + + try { + $this->expectation(new ScreenshotComparison(actual: null), $mode)->assert($path, new ToHaveScreenshotOptions(), 5000); + $this->fail('An empty baseline must fail the assertion.'); + } catch (AssertionException $e) { + $this->assertSame(sprintf('The snapshot at %s is empty. Run with PLAYWRIGHT_UPDATE_SNAPSHOTS=changed to record it again.', $path), $e->getMessage()); + } + + $this->assertSame([], $this->calls); + $this->assertSame('', file_get_contents($path)); + } + + /** + * @return iterable + */ + public static function provideComparingModes(): iterable + { + yield 'none' => ['none']; + yield 'missing' => ['missing']; + } + + #[DataProvider('provideUpdatingModes')] + public function testEmptyBaselineIsRecordedAgainWhenUpdating(string $mode): void + { + $path = $this->baseline(''); + + $this->expectation(new ScreenshotComparison(actual: 'stable'), $mode)->assert($path, new ToHaveScreenshotOptions(), 5000); + + $this->assertSame('stable', file_get_contents($path)); + $this->assertNull($this->calls[0]['expected']); + } + + public function testFailureReplacesTheImagesOfThePreviousFailure(): void + { + $path = $this->baseline('baseline'); + + foreach ([new ScreenshotComparison(actual: 'first', diff: 'diff', errorMessage: 'different'), new ScreenshotComparison(actual: 'moving', errorMessage: 'unstable', timedOut: true)] as $result) { + try { + $this->expectation($result)->assert($path, new ToHaveScreenshotOptions(), 5000); + $this->fail('A mismatch must fail the assertion.'); + } catch (AssertionException) { + } + } + + $this->assertSame(['card-actual.png', 'card-expected.png'], array_map('basename', glob($this->dir.'/out/*/*') ?: [])); + $this->assertSame('moving', file_get_contents($this->artifact('card-actual.png'))); + } + public function testUnknownUpdateModeIsRejected(): void { $this->expectException(\InvalidArgumentException::class); diff --git a/tests/Unit/Locator/LocatorTest.php b/tests/Unit/Locator/LocatorTest.php index e8308ac..cb5f224 100644 --- a/tests/Unit/Locator/LocatorTest.php +++ b/tests/Unit/Locator/LocatorTest.php @@ -288,6 +288,15 @@ public function testNormalizeReturnsALocatorOnTheResolvedSelector(): void $this->assertNotSame($this->locator, $normalized); } + public function testNormalizeKeepsThePageOfTheLocator(): void + { + $page = $this->createStub(PageInterface::class); + $locator = new Locator($this->transport, 'page1', '.element', null, null, [], $page); + $this->transport->method('send')->willReturn(['value' => 'internal:role=button[name="Save"i]']); + + $this->assertSame($page, $locator->normalize()->page()); + } + public function testNormalizeRejectsAResponseWithoutASelector(): void { $this->transport @@ -1209,10 +1218,6 @@ public function testDerivedLocatorsCarryThePageAlong(): void $this->assertSame($page, $locator->contentFrame()->locator('.inner')->page()); } - /** - * Rebinds $this->locator to the '.items' selector used by the filter and - * combinator tests, which need a different base selector than setUp(). - */ public function testExpectScreenshotSendsTheBaselineAndOptions(): void { $this->transport->expects($this->once()) @@ -1281,6 +1286,10 @@ public function testGetFrameSelector(): void $this->assertSame('iframe#preview', (new Locator($this->transport, 'page1', '.inner', 'iframe#preview'))->getFrameSelector()); } + /** + * Rebinds $this->locator to the '.items' selector used by the filter and + * combinator tests, which need a different base selector than setUp(). + */ private function useItemsLocator(): void { $this->locator = new Locator($this->transport, 'page1', '.items'); diff --git a/tests/Unit/Testing/ExpectFactoryTest.php b/tests/Unit/Testing/ExpectFactoryTest.php index eb6b00a..35bdf0d 100644 --- a/tests/Unit/Testing/ExpectFactoryTest.php +++ b/tests/Unit/Testing/ExpectFactoryTest.php @@ -207,6 +207,16 @@ public function testWindowsScreenshotPathsAreUsedAsIs(): void (new Expect($this->pageReturning('stable'), null, '/elsewhere'))->toHaveScreenshot('C:\\snapshots\\card.png'); } + public function testUncScreenshotPathsAreUsedAsIs(): void + { + $_SERVER[ScreenshotExpectation::UPDATE_ENV] = 'none'; + + $this->expectException(AssertionException::class); + $this->expectExceptionMessage('A snapshot doesn\'t exist at \\\\server\\share\\card.png.'); + + (new Expect($this->pageReturning('stable'), null, '/elsewhere'))->toHaveScreenshot('\\\\server\\share\\card.png'); + } + public function testScreenshotsWithoutSnapshotDirectoryResolveInTheWorkingDirectory(): void { $dir = $this->snapshotDir();