diff --git a/CHANGELOG.md b/CHANGELOG.md index 5546de8..db0dfa6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ ## [Unreleased] +### Added +- `toHaveScreenshot()` visual regression assertion on pages and locators, backed by Playwright's own screenshot comparison, with PNG or lossless WebP baselines + ## [1.5.0] - 2026-09-20 ### Added diff --git a/bin/lib/handlers.js b/bin/lib/handlers.js index 6210335..a2ed8f0 100644 --- a/bin/lib/handlers.js +++ b/bin/lib/handlers.js @@ -2,6 +2,40 @@ const { logger, ErrorHandler, CommandRegistry, BaseHandler, PromiseUtils, FrameU const { globalCoordinator } = require('./coordination'); const { PopupCoordinator } = require('./popup-coordinator'); +// Runs Playwright's own toHaveScreenshot machinery: the loop until two +// consecutive screenshots match, masking, and the pixelmatch or ssim-cie94 +// comparison. page._expectScreenshot() is private API, covered by the +// functional suite so a Playwright upgrade that moves it fails there first. +// The defaults below are the ones @playwright/test applies in its runner, +// which the private method does not apply by itself. +async function expectScreenshot(page, locator, command) { + const options = command.options || {}; + const mask = (options.mask || []).map(target => target.frameSelector + ? FrameUtils.resolve(page, target.frameSelector).locator(target.selector) + : page.locator(target.selector)); + + const result = await page._expectScreenshot({ + animations: 'disabled', + caret: 'hide', + scale: 'css', + ...options, + mask, + locator: locator || undefined, + expected: typeof command.expected === 'string' ? Buffer.from(command.expected, 'base64') : undefined, + isNot: false, + }); + + const encode = buffer => (buffer ? buffer.toString('base64') : null); + + return { + actual: encode(result.actual), + previous: encode(result.previous), + diff: encode(result.diff), + errorMessage: result.errorMessage || null, + timedOut: Boolean(result.timedOut), + }; +} + // The two callbacks below never run in this Node process: Playwright ships // their source to the browser, so their eval() has page scope only, exactly // like the callbacks passed to page.evaluate() elsewhere in this file. @@ -397,6 +431,7 @@ class PageHandler extends BaseHandler { waitForSelector: () => page.waitForSelector(command.selector, command.options), waitForFunction: () => this.waitForFunction(page, command), screenshot: () => PromiseUtils.wrapBinary(page.screenshot(command.options)), + expectScreenshot: () => expectScreenshot(page, null, command), pdf: () => PromiseUtils.wrapBinary(page.pdf(command.options || {})), evaluateHandle: () => this.evaluateHandle(page, command), addScriptTag: () => page.addScriptTag(command.options), @@ -778,6 +813,7 @@ class LocatorHandler extends BaseHandler { getAttribute: () => PromiseUtils.wrapValue(locator.getAttribute(command.name)), selectOption: () => PromiseUtils.wrapValues(locator.selectOption(command.values, command.options)), screenshot: () => PromiseUtils.wrapBinary(locator.screenshot(command.options)), + expectScreenshot: () => expectScreenshot(page, locator, command), evaluate: () => this.evaluateLocator(locator, command), evaluateHandle: () => this.evaluateHandle(locator, command), waitForFunction: () => this.waitForFunction(locator, command), diff --git a/docs/guide/assertions-reference.md b/docs/guide/assertions-reference.md index 7833f05..9f88ff1 100644 --- a/docs/guide/assertions-reference.md +++ b/docs/guide/assertions-reference.md @@ -80,6 +80,7 @@ These assertions are available when you pass a `Locator` to `expect()`. * **`toHaveId(string $id)`**: Asserts the element has the given ID. * **`toHaveClass(string|array $class)`**: Asserts the complete class list matches. * **`toHaveCount(int $count)`**: Asserts the locator resolves to a specific number of elements. +* **`toHaveScreenshot(string $name, ?ToHaveScreenshotOptions $options = null)`**: Compares a stable screenshot of the element with a baseline image. See [Visual Regression](visual-regression.md). ----- @@ -89,3 +90,4 @@ These assertions are available when you pass a `Page` object to `expect()`. * **`toHaveURL(string $url)`**: Asserts the page's current URL is a match. * **`toHaveTitle(string $title)`**: Asserts the page's title is a match. +* **`toHaveScreenshot(string $name, ?ToHaveScreenshotOptions $options = null)`**: Compares a stable screenshot of the page with a baseline image. See [Visual Regression](visual-regression.md). diff --git a/docs/guide/visual-regression.md b/docs/guide/visual-regression.md new file mode 100644 index 0000000..90920bd --- /dev/null +++ b/docs/guide/visual-regression.md @@ -0,0 +1,236 @@ +# Visual Regression + +`toHaveScreenshot()` compares a screenshot of a page or an element with a baseline image stored next to your tests. +It catches what the other assertions cannot see: a broken layout, a wrong color, a font that stopped loading. + +```php +public function testCardLooksTheSame(): void +{ + $this->page->goto('/design-system/card'); + + $this->expect($this->page->getByTestId('preview'))->toHaveScreenshot('card.webp'); +} +``` + +## Why compare pixels + +Every other assertion reads the DOM: `toBeVisible()`, `toHaveText()` and `toHaveCSS()` check one element and one property +at a time. What the user sees is the rendered result of all of them together, and the two can disagree. A page where +every element is attached, visible and correctly labelled can still look broken: only the rendered pixels tell. + +A screenshot comparison checks that rendered result as a whole. It catches the bugs no DOM assertion is written for: + +- **CSS side effects.** A margin changed in a shared stylesheet, or a design token renamed, and a card three pages away + overflows its container. +- **Design system drift.** A button variant loses its border radius, the dark theme picks the wrong background, an icon + moves by 4 pixels. +- **Assets and fonts.** A web font fails to load and the fallback reflows the page, an SVG icon goes missing, an image + is stretched. +- **Layout at breakpoints.** Text wraps differently, elements overlap, a sticky header covers content on a narrow + viewport. +- **Upgrades.** A new version of a CSS framework, a UI component library or the browser itself, with every visual + consequence visible in one run. + +Describing "this looks right" with DOM assertions would take hundreds of `toHaveCSS()` checks per component, and they +would still miss how elements interact. One screenshot covers it, and the diff image points at the exact place that +changed: cheap to write, fast to review. + +### A defensive test, on chosen surfaces + +Visual regression is a guard against unwanted change. It belongs on the few surfaces whose appearance is part of the +contract: the design system, the pages that carry the product. Every baseline is a file to review each time the +design moves on purpose, so screenshotting every page of an application turns each redesign into a chore and each +failure into noise. A handful of well-chosen baselines stays meaningful: when one fails, something that mattered +changed. + +### Where it pays off + +- **A component gallery.** One page renders each component in every variant, one screenshot per component and theme. + A data provider turns it into a matrix: + + ```php + #[DataProvider('components')] + public function testComponentLooksTheSame(string $name, string $theme): void + { + $this->page->emulateMedia(['colorScheme' => $theme]); + $this->page->goto('/design-system/'.$name); + + $this->expect($this->page->getByTestId('preview'))->toHaveScreenshot(sprintf('%s-%s.webp', $name, $theme)); + } + + public static function components(): iterable + { + foreach (['button', 'badge', 'alert', 'card', 'modal'] as $name) { + foreach (['light', 'dark'] as $theme) { + yield sprintf('%s %s', $name, $theme) => [$name, $theme]; + } + } + } + ``` + +- **Precious pages.** Login, checkout, landing pages: the screens where a visual break costs money or trust. +- **Refactors and upgrades.** Record the baselines, change the code, and see exactly what moved. + +### What it complements + +A screenshot checks appearance. Clicking, submitting and navigating stay the job of behavior assertions, which prove +the application works; visual regression proves it still looks the way it was designed. Pages full of dynamic content +(dates, avatars, ads, user data) need masks or fixed fixtures to compare reliably, and baselines need one reference +system, since fonts render differently from one system to another. Both are covered below. + +## How it works + +The comparison runs in Playwright itself, with the same code as `toHaveScreenshot()` in `@playwright/test`: + +1. Playwright takes screenshots until two consecutive ones are identical. Animations are disabled, the text caret is + hidden and the image uses CSS pixels, so the page has a chance to settle. +2. The stable screenshot is compared with the baseline, pixel by pixel, within the tolerance you configure. +3. When they differ, the expected, actual and diff images are written to the test's own directory under + `test-failures/`, and the assertion fails with their paths. + +## Baselines + +The first run has nothing to compare with: it writes the baseline and fails, so a new screenshot is always reviewed +before it is trusted. Run the test again and it passes. + +A relative name is stored next to the test file, suffixed with the browser and the platform, like `@playwright/test` +does. Both are read from the page under test, nothing to configure. Browsers and systems render fonts differently, so +each combination keeps its own baseline: + +``` +tests/E2E/CardTest.php +tests/E2E/CardTest.php-snapshots/card-chromium-darwin.webp +tests/E2E/CardTest.php-snapshots/card-chromium-linux.webp +tests/E2E/CardTest.php-snapshots/card-firefox-linux.webp +``` + +A persistent context has no browser object: its baselines carry the platform only. An absolute path is used as is. To +store baselines elsewhere, override `snapshotDirectory()` in your test case. + +Commit the baselines. Generate the ones used in CI on the same system as CI, ideally in the same Docker image. + +Record and compare in headless mode, the default and what CI runs. Both headless modes of Chromium, the headless shell +and `channel: 'chromium'`, produce byte-identical screenshots. A visible browser shows a scrollbar as soon as the page +scrolls, 15 pixels that fail any full-page comparison against a headless baseline. + +### Failure images + +Each test writes its failure images in its own directory, one per browser, as Playwright JS does per test and project +under `test-results/`: `test-failures/--/`. Override `testOutputDirectory()` in your +test case to move it. Each baseline has a subdirectory with its name and a short identifier derived from its path. +This keeps `light/card.png`, `dark/card.png` and `card.webp` separate, including when a passing assertion cleans up +its earlier failure. The image names omit the browser, which the test directory already identifies. +For `toHaveScreenshot('card.png')` in `CardTest::testCard`, run in Chromium, the baseline is +`card-chromium-linux.png` and a failure writes to `test-failures/CardTest-testCard-chromium/card-/`: + +| File | Content | +|---|---| +| `card-expected.png` | Copy of the baseline at the time of the failure | +| `card-actual.png` | Last screenshot taken | +| `card-diff.png` | Baseline in pale grey, differing pixels in red. Always a PNG | + +Expected and actual keep the format of the baseline, `.webp` included. The three files are removed as soon as the +baseline matches again or is recorded again, and the directory with them once empty: a green run leaves a clean +`test-failures/`. + +What happens to the baseline itself: + +| Situation | `missing` (default) | `none` | `changed` | `all` | +|---|---|---|---|---| +| No baseline | written, test fails | test fails | written, test passes | written, test passes | +| Baseline matches | kept, test passes | kept, test passes | kept, test passes | rewritten, test passes | +| Baseline differs | kept, failure images, test fails | kept, failure images, test fails | rewritten, test passes | rewritten, test passes | +| Page never stable, baseline present | kept, failure images, test fails | kept, failure images, test fails | kept, failure images, test fails | kept, test fails | +| Page never stable, no baseline | nothing written, test fails | nothing written, test fails | nothing written, test fails | nothing written, test fails | + +### Prefer WebP + +The extension of the name picks the format. Without extension, `.png` is added, as in `@playwright/test`. We recommend +`.webp`: Playwright records it lossless, pixel for pixel the same image as the PNG. + +```php +$this->expect($this->page)->toHaveScreenshot('dashboard.webp'); +``` + +| | PNG | WebP | +|---|---|---| +| Size, flat interface | 12 KB | 3 KB | +| Size, viewport with gradients | 357 KB | 24 KB | +| Comparison, viewport | 88 ms | 21 ms | +| Comparison, full page | 423 ms | 153 ms | + +A repository that stores hundreds of baselines stays light, and the suite runs faster since decoding dominates the +comparison. PNG keeps one advantage: more review tools preview it in a diff. The diff image of a failure is always a PNG. +Other extensions, JPEG included, throw an `InvalidArgumentException`: only lossless formats compare reliably. + +### Updating baselines + +The `PLAYWRIGHT_UPDATE_SNAPSHOTS` variable takes the values of the `--update-snapshots` flag of `@playwright/test`: + +| Value | Missing baseline | Different screenshot | +|---|---|---| +| `missing` (default) | written, the test fails | the test fails | +| `none` | the test fails | the test fails | +| `changed` | written, the test passes | rewritten, the test passes | +| `all` | written, the test passes | rewritten, matching ones too | + +```shell +PLAYWRIGHT_UPDATE_SNAPSHOTS=changed vendor/bin/phpunit tests/E2E +``` + +Then review the changed images in your diff before committing them. + +## Options + +Pass a `ToHaveScreenshotOptions` with named arguments to tune the comparison: + +```php +use Playwright\Assertions\Options\ToHaveScreenshotOptions; + +$this->expect($this->page)->toHaveScreenshot('dashboard.webp', new ToHaveScreenshotOptions( + fullPage: true, + maxDiffPixelRatio: 0.01, + mask: [$this->page->locator('.timestamp'), $this->page->locator('.avatar')], +)); +``` + +The defaults are the ones of `@playwright/test`: + +| Option | Default | Effect | +|---|---|---| +| `threshold` | `0.2` | Color distance tolerated for each pixel, from 0 (exact) to 1. Used by pixelmatch only | +| `maxDiffPixels` | none | Number of differing pixels tolerated | +| `maxDiffPixelRatio` | none | Share of differing pixels tolerated, from 0 to 1 | +| `comparator` | `'pixelmatch'` | `'ssim-cie94'` compares structure and perceived color, experimental in Playwright | +| `mask` | `[]` | Locators covered with a solid box, for dates, avatars, ads. Inside an iframe, use `frameLocator()` | +| `maskColor` | `'#FF00FF'` | Any CSS color | +| `style` | none | CSS applied while taking the screenshot, to hide or freeze dynamic parts | +| `fullPage` | `false` | Captures the whole scrollable page. Pages only | +| `clip` | none | Region to capture: `['x' => 0, 'y' => 0, 'width' => 320, 'height' => 200]`. Pages only | +| `omitBackground` | `false` | Transparent background instead of white | +| `animations` | `'disabled'` | Finite animations jump to their end, infinite ones are cancelled. `'allow'` keeps them running | +| `caret` | `'hide'` | `'initial'` keeps the blinking text caret | +| `scale` | `'css'` | One pixel per CSS pixel, identical on every screen. `'device'` is sharper on high-DPI screens | +| `timeoutMs` | assertion timeout | Time allowed to get two identical consecutive screenshots | +| `message` | generated | Replaces the first line of the failure message | + +Invalid values throw an `InvalidArgumentException` when the options are created, so a typo such as `comparator: 'ssim'` +fails in PHP with the list of accepted values. Numeric options must be finite: `NAN` and infinities are rejected before +reaching the browser. On a locator, `fullPage` and `clip` throw too: an element is captured whole. + +### Tolerating differences + +Without tolerance, a single differing pixel fails the assertion. `maxDiffPixels` suits a fixed-size element, +`maxDiffPixelRatio` a page whose size varies. When both are set, the stricter one applies. + +### Choosing a threshold + +The default `threshold` of `0.2` forgives anti-aliasing and small rendering noise. It also forgives close shades of one +hue: `#818cf8` and `#6366f1`, two indigos of the same palette, compare as identical. To catch a design token change, lower +it to `0.1` or switch to the `ssim-cie94` comparator, which ignores `threshold`. + +## Limits + +- Negation throws a `LogicException`: a screenshot either matches its baseline or is reviewed. +- Decorated pages and test doubles throw a `LogicException` too. The comparison needs the `Page` and `Locator` created + by Playwright PHP. diff --git a/src/Assertions/Internal/AbstractAssertions.php b/src/Assertions/Internal/AbstractAssertions.php index 87ff4a8..a56fde3 100644 --- a/src/Assertions/Internal/AbstractAssertions.php +++ b/src/Assertions/Internal/AbstractAssertions.php @@ -72,6 +72,33 @@ protected function assertCondition( } } + /** + * Runs a matcher that brings its own waiting, such as a screenshot + * comparison, inside the same tracing group as polled matchers. + * + * @param callable(int): void $assertion receives the timeout in milliseconds + */ + protected function assertWithoutPolling(string $matcher, ?int $timeoutMs, callable $assertion): void + { + if ($this->negated) { + $this->negated = false; + + throw new \LogicException(sprintf('%s() cannot be negated.', $matcher)); + } + + if (null !== $this->tracing) { + $this->tracing->group(sprintf('expect(%s).%s', $this->subjectName(), $matcher)); + } + + try { + $assertion($timeoutMs ?? $this->timeoutMs); + } finally { + if (null !== $this->tracing) { + $this->tracing->groupEnd(); + } + } + } + abstract protected function subjectName(): string; /** diff --git a/src/Assertions/Internal/ScreenshotExpectation.php b/src/Assertions/Internal/ScreenshotExpectation.php new file mode 100644 index 0000000..30a4b62 --- /dev/null +++ b/src/Assertions/Internal/ScreenshotExpectation.php @@ -0,0 +1,240 @@ +): ScreenshotComparison $compare + */ + public function __construct( + private readonly \Closure $compare, + private readonly string $subject, + private readonly ?string $updateMode = null, + private readonly ?string $outputDir = null, + private readonly ?string $variant = null, + ) { + } + + public function assert(string $path, ToHaveScreenshotOptions $options, int $timeoutMs): void + { + $mode = $this->updateMode(); + $payload = $options->toArray() + ['timeout' => $options->timeoutMs ?? $timeoutMs, 'type' => $this->imageType($path)]; + + $expected = is_file($path) ? @file_get_contents($path) : null; + if (false === $expected) { + throw new \RuntimeException(sprintf('Unable to read the snapshot "%s".', $path)); + } + + // An empty file would reach Playwright as "no baseline" and match + // anything: it only passes once recorded again. + if ('' === $expected && 'changed' !== $mode && 'all' !== $mode) { + throw new AssertionException($options->message ?? sprintf('The snapshot at %s is empty. Run with %s=changed to record it again.', $path, self::UPDATE_ENV)); + } + + // "all" records every screenshot, like --update-snapshots=all: no + // comparison, Playwright returns no image when a comparison matches. + // "changed" records an empty baseline the same way. + if (null === $expected || '' === $expected || 'all' === $mode) { + $this->record($path, $mode, null === $expected, ($this->compare)(null, $payload), $options); + + return; + } + + $result = ($this->compare)($expected, $payload); + + if ($result->matches()) { + $this->removeArtifacts($path); + + return; + } + + if ('changed' === $mode && null !== $result->actual && !$result->timedOut) { + $this->write($path, $result->actual); + $this->removeArtifacts($path); + + return; + } + + $artifacts = $this->writeArtifacts($path, $expected, $result); + + throw new AssertionException($this->failureMessage($options, $result, $path, $artifacts), actual: $artifacts['actual'] ?? null, expected: $path); + } + + private function record(string $path, string $mode, bool $missing, ScreenshotComparison $result, ToHaveScreenshotOptions $options): void + { + if (null === $result->actual || !$result->matches()) { + throw new AssertionException($options->message ?? sprintf('Failed to take a stable screenshot of the %s: %s', $this->subject, $result->errorMessage ?? 'no screenshot returned')); + } + + if ($missing && 'none' === $mode) { + throw new AssertionException($options->message ?? sprintf('A snapshot doesn\'t exist at %s. Run with %s=missing to write it.', $path, self::UPDATE_ENV)); + } + + $this->write($path, $result->actual); + $this->removeArtifacts($path); + + if ($missing && 'missing' === $mode) { + throw new AssertionException($options->message ?? sprintf('A snapshot doesn\'t exist at %s, writing actual.', $path)); + } + } + + /** + * Images of an earlier failure would read as a current one: they go as + * soon as the baseline matches or is recorded again. + */ + private function removeArtifacts(string $path): void + { + $stem = $this->artifactStem($path); + $extension = '.'.$this->imageType($path); + + foreach ([$stem.'-expected'.$extension, $stem.'-actual'.$extension, $stem.'-diff.png'] as $file) { + if (is_file($file)) { + unlink($file); + } + } + + foreach ([dirname($stem), $this->outputDir()] as $directory) { + if (is_dir($directory) && [] === array_diff(scandir($directory) ?: [], ['.', '..'])) { + rmdir($directory); + } + } + } + + /** + * @return array<'expected'|'actual'|'diff', string> + */ + private function writeArtifacts(string $path, string $expected, ScreenshotComparison $result): array + { + $this->removeArtifacts($path); + + $stem = $this->artifactStem($path); + $extension = '.'.$this->imageType($path); + + $artifacts = ['expected' => $stem.'-expected'.$extension]; + $this->write($artifacts['expected'], $expected); + + if (null !== $result->actual) { + $artifacts['actual'] = $stem.'-actual'.$extension; + $this->write($artifacts['actual'], $result->actual); + } + if (null !== $result->diff) { + $artifacts['diff'] = $stem.'-diff.png'; + $this->write($artifacts['diff'], $result->diff); + } + + return $artifacts; + } + + /** + * @param array<'expected'|'actual'|'diff', string> $artifacts + */ + private function failureMessage(ToHaveScreenshotOptions $options, ScreenshotComparison $result, string $path, array $artifacts): string + { + $message = $options->message ?? sprintf('Screenshot of the %s does not match %s.', $this->subject, $path); + + $lines = [$message, (string) $result->errorMessage]; + foreach ($artifacts as $kind => $file) { + $lines[] = sprintf(' %-9s %s', $kind.':', $file); + } + $lines[] = sprintf('Run with %s=changed to accept the new screenshot.', self::UPDATE_ENV); + + return implode("\n", $lines); + } + + /** + * The baseline extension picks the image format. WebP is lossless here, + * smaller and faster to compare; JPEG is rejected, its compression noise + * would fail every comparison. + * + * @return 'png'|'webp' + */ + private function imageType(string $path): string + { + return match (strtolower(pathinfo($path, \PATHINFO_EXTENSION))) { + 'png' => 'png', + 'webp' => 'webp', + default => throw new \InvalidArgumentException(sprintf('Snapshot "%s" must be a .png or .webp file.', $path)), + }; + } + + private function updateMode(): string + { + $mode = $this->updateMode ?? $_SERVER[self::UPDATE_ENV] ?? getenv(self::UPDATE_ENV); + if (!is_string($mode) || '' === $mode) { + return 'missing'; + } + + if (!in_array($mode, self::UPDATE_MODES, true)) { + throw new \InvalidArgumentException(sprintf('%s must be one of "%s", "%s" given.', self::UPDATE_ENV, implode('", "', self::UPDATE_MODES), $mode)); + } + + return $mode; + } + + /** + * Failure images drop the variant of the baseline name, which their + * directory already carries: "card-chromium-linux.png" fails as + * "card-actual.png", as in Playwright JS. Each baseline gets its own + * directory so matching one cannot erase another baseline's failure. + */ + private function artifactStem(string $path): string + { + $stem = pathinfo($path, \PATHINFO_FILENAME); + if (null !== $this->variant && '' !== $this->variant && str_ends_with($stem, '-'.$this->variant)) { + $stem = substr($stem, 0, -\strlen($this->variant) - 1); + } + + $identity = substr(hash('sha256', realpath($path) ?: $path), 0, 12); + + return $this->outputDir().'/'.$stem.'-'.$identity.'/'.$stem; + } + + private function outputDir(): string + { + return $this->outputDir ?? getcwd().'/test-failures/snapshots'; + } + + private function write(string $file, string $contents): void + { + $dir = dirname($file); + if (!is_dir($dir) && !@mkdir($dir, 0777, true) && !is_dir($dir)) { + throw new \RuntimeException(sprintf('Directory "%s" was not created.', $dir)); + } + + if (false === @file_put_contents($file, $contents)) { + throw new \RuntimeException(sprintf('Unable to write "%s".', $file)); + } + } +} diff --git a/src/Assertions/LocatorAssertions.php b/src/Assertions/LocatorAssertions.php index 28710a2..f7576d9 100644 --- a/src/Assertions/LocatorAssertions.php +++ b/src/Assertions/LocatorAssertions.php @@ -16,6 +16,9 @@ use Playwright\Assertions\Internal\AbstractAssertions; use Playwright\Assertions\Internal\AriaSnapshot; +use Playwright\Assertions\Internal\ScreenshotExpectation; +use Playwright\Assertions\Options\ToHaveScreenshotOptions; +use Playwright\Locator\Locator; use Playwright\Locator\LocatorInterface; use Playwright\Tracing\TracingInterface; @@ -83,9 +86,17 @@ final class LocatorAssertions extends AbstractAssertions implements LocatorAsser } JS; + /** + * @param string|null $screenshotOutputDir Where toHaveScreenshot() writes the expected, actual and diff images of a + * failure. Defaults to test-failures/snapshots/ in the working directory. + * @param string|null $screenshotVariant Suffix of the baseline names, such as "chromium-linux", left out of the + * failure image names: "card-chromium-linux.png" fails as "card-actual.png". + */ public function __construct( private readonly LocatorInterface $locator, ?TracingInterface $tracing = null, + private readonly ?string $screenshotOutputDir = null, + private readonly ?string $screenshotVariant = null, ) { parent::__construct($tracing); } @@ -630,6 +641,33 @@ private static function classTokens(string $classAttribute): array return false === $tokens ? [] : $tokens; } + public function toHaveScreenshot(string $path, ?ToHaveScreenshotOptions $options = null): self + { + $options ??= new ToHaveScreenshotOptions(); + $subject = $this->locator; + if (!$subject instanceof Locator) { + throw new \LogicException(sprintf('toHaveScreenshot() needs a %s created by Playwright PHP, %s given.', Locator::class, get_debug_type($subject))); + } + + if ($options->fullPage || null !== $options->clip) { + throw new \InvalidArgumentException('The fullPage and clip options apply to a page screenshot, an element is captured whole.'); + } + + $outputDir = $this->screenshotOutputDir; + $variant = $this->screenshotVariant; + $this->assertWithoutPolling('toHaveScreenshot', $options->timeoutMs, static function (int $timeoutMs) use ($subject, $path, $options, $outputDir, $variant): void { + $expectation = new ScreenshotExpectation( + $subject->expectScreenshot(...), + 'locator', + outputDir: $outputDir, + variant: $variant, + ); + $expectation->assert($path, $options, $timeoutMs); + }); + + return $this; + } + protected function subjectName(): string { return $this->locator->getSelector(); diff --git a/src/Assertions/LocatorAssertionsInterface.php b/src/Assertions/LocatorAssertionsInterface.php index 69de067..3354b1c 100644 --- a/src/Assertions/LocatorAssertionsInterface.php +++ b/src/Assertions/LocatorAssertionsInterface.php @@ -14,6 +14,8 @@ namespace Playwright\Assertions; +use Playwright\Assertions\Options\ToHaveScreenshotOptions; + interface LocatorAssertionsInterface { /** @@ -147,6 +149,21 @@ public function toHaveAccessibleErrorMessage(string $expected, ?AssertionOptions */ public function toMatchAriaSnapshot(string $expected, ?AssertionOptions $options = null): self; + /** + * Asserts a stable screenshot of the element matches the baseline image at + * $path, a .png or a lossless .webp. + * + * Screenshots are taken until two consecutive ones are identical, then + * compared with the baseline by Playwright's own comparator. A missing + * baseline is written and the assertion fails; PLAYWRIGHT_UPDATE_SNAPSHOTS + * ("none", "missing", "changed", "all") controls what gets rewritten. On + * failure, the expected, actual and diff images are written to the + * screenshot output directory, test-failures/snapshots/ by default, and + * removed once the baseline matches or is recorded again. Negation throws + * a LogicException. + */ + public function toHaveScreenshot(string $path, ?ToHaveScreenshotOptions $options = null): self; + public function not(): self; public function withTimeout(int $timeoutMs): self; diff --git a/src/Assertions/Options/ToHaveScreenshotOptions.php b/src/Assertions/Options/ToHaveScreenshotOptions.php new file mode 100644 index 0000000..29baea0 --- /dev/null +++ b/src/Assertions/Options/ToHaveScreenshotOptions.php @@ -0,0 +1,197 @@ +locator('.date')]) + * + * Invalid values throw an \InvalidArgumentException on construction, so a + * typo fails in PHP before reaching the browser. + */ +final readonly class ToHaveScreenshotOptions +{ + public const COMPARATOR_PIXELMATCH = 'pixelmatch'; + public const COMPARATOR_SSIM_CIE94 = 'ssim-cie94'; + + private const COMPARATORS = [self::COMPARATOR_PIXELMATCH, self::COMPARATOR_SSIM_CIE94]; + private const ANIMATIONS = ['disabled', 'allow']; + private const CARETS = ['hide', 'initial']; + private const SCALES = ['css', 'device']; + + /** + * @param float $threshold Color distance tolerated for each pixel, from 0 (exact) to 1 (anything). + * Used by pixelmatch only. 0.2 forgives anti-aliasing, and also close + * shades of one hue: lower it to 0.1 to catch a design token change. + * @param int|null $maxDiffPixels Number of differing pixels tolerated. None by default: any differing + * pixel fails. When set with $maxDiffPixelRatio, the stricter one wins. + * @param float|null $maxDiffPixelRatio Share of differing pixels tolerated, from 0 to 1. None by default. + * @param 'pixelmatch'|'ssim-cie94' $comparator pixelmatch compares pixel colors. ssim-cie94 compares local structure + * and perceived color: steadier across rendering noise, stricter on + * color shifts, slower, experimental in Playwright. + * @param list $mask Elements covered with a solid box of $maskColor before the comparison, + * for dates, avatars or ads. A locator obtained through frameLocator() + * masks inside its iframe. + * @param string $maskColor Any CSS color. Magenta by default, a color no interface uses. + * @param string|null $style CSS applied while the screenshot is taken, to hide or freeze dynamic + * parts. It pierces the Shadow DOM and applies to iframes. + * @param bool $fullPage Captures the whole scrollable page. Page assertions only. + * @param array{x: float, y: float, width: float, height: float}|null $clip region of the page to capture, in CSS pixels + * @param bool $omitBackground transparent background instead of the default white one + * @param 'disabled'|'allow' $animations "disabled" fast-forwards finite CSS animations and transitions to their + * end and cancels infinite ones, so the page can settle + * @param 'hide'|'initial' $caret "hide" hides the text caret, which blinks + * @param 'css'|'device' $scale "css" takes one pixel per CSS pixel, so baselines match across screen + * densities. "device" is sharper and larger on high-DPI screens. + * @param int|null $timeoutMs Time allowed to get two identical consecutive screenshots, in + * milliseconds. Defaults to the assertion timeout. + * @param string|null $message replaces the first line of the failure message + */ + public function __construct( + public float $threshold = 0.2, + public ?int $maxDiffPixels = null, + public ?float $maxDiffPixelRatio = null, + public string $comparator = self::COMPARATOR_PIXELMATCH, + public array $mask = [], + public string $maskColor = '#FF00FF', + public ?string $style = null, + public bool $fullPage = false, + public ?array $clip = null, + public bool $omitBackground = false, + public string $animations = 'disabled', + public string $caret = 'hide', + public string $scale = 'css', + public ?int $timeoutMs = null, + public ?string $message = null, + ) { + self::assertRange('threshold', $threshold); + if (null !== $maxDiffPixelRatio) { + self::assertRange('maxDiffPixelRatio', $maxDiffPixelRatio); + } + if (null !== $maxDiffPixels && $maxDiffPixels < 0) { + throw new \InvalidArgumentException(sprintf('The maxDiffPixels option must be 0 or more, %d given.', $maxDiffPixels)); + } + if (null !== $timeoutMs && $timeoutMs < 0) { + throw new \InvalidArgumentException(sprintf('The timeoutMs option must be 0 or more, %d given.', $timeoutMs)); + } + + self::assertChoice('comparator', $comparator, self::COMPARATORS); + self::assertChoice('animations', $animations, self::ANIMATIONS); + self::assertChoice('caret', $caret, self::CARETS); + self::assertChoice('scale', $scale, self::SCALES); + + if ('' === trim($maskColor)) { + throw new \InvalidArgumentException('The maskColor option must be a CSS color, an empty string given.'); + } + + foreach ($mask as $index => $locator) { + if (!$locator instanceof LocatorInterface) { + throw new \InvalidArgumentException(sprintf('The mask option expects %s instances, %s given at index %s.', LocatorInterface::class, get_debug_type($locator), $index)); + } + } + + if (null !== $clip) { + self::assertClip($clip); + } + } + + /** + * Options sent to the bridge. The timeout and the message are applied on + * the PHP side and are not part of it. + * + * @return array + */ + public function toArray(): array + { + $options = array_filter([ + 'threshold' => $this->threshold, + 'maxDiffPixels' => $this->maxDiffPixels, + 'maxDiffPixelRatio' => $this->maxDiffPixelRatio, + 'comparator' => $this->comparator, + 'maskColor' => $this->maskColor, + 'style' => $this->style, + 'fullPage' => $this->fullPage, + 'clip' => $this->clip, + 'omitBackground' => $this->omitBackground, + 'animations' => $this->animations, + 'caret' => $this->caret, + 'scale' => $this->scale, + ], static fn (mixed $value): bool => null !== $value); + + if ([] !== $this->mask) { + $options['mask'] = array_map( + static fn (LocatorInterface $locator): array => array_filter([ + 'selector' => $locator->getSelector(), + 'frameSelector' => $locator instanceof Locator ? $locator->getFrameSelector() : null, + ], static fn (?string $value): bool => null !== $value), + $this->mask, + ); + } + + return $options; + } + + private static function assertRange(string $option, float $value): void + { + if (!is_finite($value)) { + throw new \InvalidArgumentException(sprintf('The %s option must be between 0 and 1, a non-finite value given.', $option)); + } + + if ($value < 0 || $value > 1) { + throw new \InvalidArgumentException(sprintf('The %s option must be between 0 and 1, %s given.', $option, $value)); + } + } + + /** + * @param list $choices + */ + private static function assertChoice(string $option, string $value, array $choices): void + { + if (!in_array($value, $choices, true)) { + throw new \InvalidArgumentException(sprintf('The %s option must be one of "%s", "%s" given.', $option, implode('", "', $choices), $value)); + } + } + + /** + * @param array $clip + */ + private static function assertClip(array $clip): void + { + foreach (['x', 'y', 'width', 'height'] as $key) { + if (!isset($clip[$key]) || !is_int($clip[$key]) && !is_float($clip[$key])) { + throw new \InvalidArgumentException(sprintf('The clip option needs numeric "x", "y", "width" and "height" keys, "%s" is missing or not a number.', $key)); + } + + if (!is_finite($clip[$key])) { + throw new \InvalidArgumentException(sprintf('The clip option needs a finite "%s" value.', $key)); + } + } + + if ($clip['width'] <= 0 || $clip['height'] <= 0) { + throw new \InvalidArgumentException('The clip option needs a width and a height greater than 0.'); + } + + if ([] !== array_diff(array_keys($clip), ['x', 'y', 'width', 'height'])) { + throw new \InvalidArgumentException(sprintf('The clip option only accepts "x", "y", "width" and "height" keys, "%s" given.', implode('", "', array_keys($clip)))); + } + } +} diff --git a/src/Assertions/PageAssertions.php b/src/Assertions/PageAssertions.php index a4ce047..eadfa27 100644 --- a/src/Assertions/PageAssertions.php +++ b/src/Assertions/PageAssertions.php @@ -16,14 +16,25 @@ use Playwright\Assertions\Internal\AbstractAssertions; use Playwright\Assertions\Internal\AriaSnapshot; +use Playwright\Assertions\Internal\ScreenshotExpectation; +use Playwright\Assertions\Options\ToHaveScreenshotOptions; +use Playwright\Page\Page; use Playwright\Page\PageInterface; use Playwright\Tracing\TracingInterface; final class PageAssertions extends AbstractAssertions implements PageAssertionsInterface { + /** + * @param string|null $screenshotOutputDir Where toHaveScreenshot() writes the expected, actual and diff images of a + * failure. Defaults to test-failures/snapshots/ in the working directory. + * @param string|null $screenshotVariant Suffix of the baseline names, such as "chromium-linux", left out of the + * failure image names: "card-chromium-linux.png" fails as "card-actual.png". + */ public function __construct( private readonly PageInterface $page, ?TracingInterface $tracing = null, + private readonly ?string $screenshotOutputDir = null, + private readonly ?string $screenshotVariant = null, ) { parent::__construct($tracing); } @@ -96,6 +107,29 @@ public function toMatchAriaSnapshot(string $expected, ?AssertionOptions $options return $this; } + public function toHaveScreenshot(string $path, ?ToHaveScreenshotOptions $options = null): self + { + $options ??= new ToHaveScreenshotOptions(); + $subject = $this->page; + if (!$subject instanceof Page) { + throw new \LogicException(sprintf('toHaveScreenshot() needs a %s created by Playwright PHP, %s given.', Page::class, get_debug_type($subject))); + } + + $outputDir = $this->screenshotOutputDir; + $variant = $this->screenshotVariant; + $this->assertWithoutPolling('toHaveScreenshot', $options->timeoutMs, static function (int $timeoutMs) use ($subject, $path, $options, $outputDir, $variant): void { + $expectation = new ScreenshotExpectation( + $subject->expectScreenshot(...), + 'page', + outputDir: $outputDir, + variant: $variant, + ); + $expectation->assert($path, $options, $timeoutMs); + }); + + return $this; + } + protected function subjectName(): string { return 'page'; diff --git a/src/Assertions/PageAssertionsInterface.php b/src/Assertions/PageAssertionsInterface.php index 5f534cb..24c531f 100644 --- a/src/Assertions/PageAssertionsInterface.php +++ b/src/Assertions/PageAssertionsInterface.php @@ -14,6 +14,8 @@ namespace Playwright\Assertions; +use Playwright\Assertions\Options\ToHaveScreenshotOptions; + interface PageAssertionsInterface { /** @return $this */ @@ -33,6 +35,21 @@ public function toHaveURL(string|\Stringable $expected, ?AssertionOptions $optio */ public function toMatchAriaSnapshot(string $expected, ?AssertionOptions $options = null): self; + /** + * Asserts a stable screenshot of the page matches the baseline image at + * $path, a .png or a lossless .webp. + * + * Screenshots are taken until two consecutive ones are identical, then + * compared with the baseline by Playwright's own comparator. A missing + * baseline is written and the assertion fails; PLAYWRIGHT_UPDATE_SNAPSHOTS + * ("none", "missing", "changed", "all") controls what gets rewritten. On + * failure, the expected, actual and diff images are written to the + * screenshot output directory, test-failures/snapshots/ by default, and + * removed once the baseline matches or is recorded again. Negation throws + * a LogicException. + */ + public function toHaveScreenshot(string $path, ?ToHaveScreenshotOptions $options = null): self; + /** @return $this */ public function not(): self; diff --git a/src/Locator/Locator.php b/src/Locator/Locator.php index b9acc17..a3effc3 100644 --- a/src/Locator/Locator.php +++ b/src/Locator/Locator.php @@ -52,6 +52,7 @@ use Playwright\Locator\Options\WaitForFunctionOptions; use Playwright\Locator\Options\WaitForOptions; use Playwright\Page\PageInterface; +use Playwright\Screenshot\ScreenshotComparison; use Playwright\Transport\TransportInterface; use Psr\Log\LoggerInterface; use Psr\Log\NullLogger; @@ -232,6 +233,32 @@ public function screenshot(?string $path = null, array|LocatorScreenshotOptions return is_string($binary) ? $binary : null; } + /** + * Takes screenshots until two consecutive ones match, then compares the + * last one with $expected. Without $expected, returns the stable screenshot. + * + * @internal Backs toHaveScreenshot(); the shape may change without notice + * + * @param array $options + */ + public function expectScreenshot(?string $expected, array $options = []): ScreenshotComparison + { + $response = $this->sendCommand('locator.expectScreenshot', [ + 'expected' => null === $expected ? null : base64_encode($expected), + 'options' => $options, + ]); + + return ScreenshotComparison::fromResponse($response); + } + + /** + * @internal + */ + public function getFrameSelector(): ?string + { + return $this->frameSelector; + } + /** * @return array */ @@ -995,7 +1022,7 @@ public function normalize(): self throw new ProtocolErrorException('Invalid normalize response', 0); } - return new self($this->transport, $this->pageId, $value, $this->frameSelector, $this->logger); + return new self($this->transport, $this->pageId, $value, $this->frameSelector, $this->logger, [], $this->page); } public function contentFrame(): FrameLocatorInterface diff --git a/src/Page/Page.php b/src/Page/Page.php index 3a35201..a01e425 100644 --- a/src/Page/Page.php +++ b/src/Page/Page.php @@ -73,6 +73,7 @@ use Playwright\Regex; use Playwright\Screencast\Screencast; use Playwright\Screencast\ScreencastInterface; +use Playwright\Screenshot\ScreenshotComparison; use Playwright\Screenshot\ScreenshotHelper; use Playwright\Transport\TransportInterface; use Playwright\Video\Video; @@ -396,6 +397,24 @@ public function screenshot(?string $path = null, array|ScreenshotOptions $option return $finalPath; } + /** + * Takes screenshots until two consecutive ones match, then compares the + * last one with $expected. Without $expected, returns the stable screenshot. + * + * @internal Backs toHaveScreenshot(); the shape may change without notice + * + * @param array $options + */ + public function expectScreenshot(?string $expected, array $options = []): ScreenshotComparison + { + $response = $this->sendCommand('expectScreenshot', [ + 'expected' => null === $expected ? null : base64_encode($expected), + 'options' => $options, + ]); + + return ScreenshotComparison::fromResponse($response); + } + /** * Take an auto-generated screenshot with custom suffix. * diff --git a/src/Screenshot/ScreenshotComparison.php b/src/Screenshot/ScreenshotComparison.php new file mode 100644 index 0000000..b1fa4ac --- /dev/null +++ b/src/Screenshot/ScreenshotComparison.php @@ -0,0 +1,64 @@ + $response + */ + public static function fromResponse(array $response): self + { + $binary = static function (mixed $value): ?string { + if (!is_string($value) || '' === $value) { + return null; + } + + $decoded = base64_decode($value, true); + + return false === $decoded ? null : $decoded; + }; + + $errorMessage = $response['errorMessage'] ?? null; + + return new self( + actual: $binary($response['actual'] ?? null), + diff: $binary($response['diff'] ?? null), + previous: $binary($response['previous'] ?? null), + errorMessage: is_string($errorMessage) && '' !== $errorMessage ? $errorMessage : null, + timedOut: true === ($response['timedOut'] ?? false), + ); + } + + public function matches(): bool + { + return null === $this->errorMessage; + } +} diff --git a/src/Testing/Expect.php b/src/Testing/Expect.php index 8e2279e..1ab1805 100644 --- a/src/Testing/Expect.php +++ b/src/Testing/Expect.php @@ -16,8 +16,10 @@ use Playwright\Assertions\LocatorAssertions; use Playwright\Assertions\LocatorAssertionsInterface; +use Playwright\Assertions\Options\ToHaveScreenshotOptions; use Playwright\Assertions\PageAssertions; use Playwright\Assertions\PageAssertionsInterface; +use Playwright\Exception\RuntimeException; use Playwright\Locator\LocatorInterface; use Playwright\Page\PageInterface; use Playwright\Tracing\TracingInterface; @@ -26,11 +28,32 @@ final class Expect implements ExpectInterface { private readonly LocatorAssertionsInterface|PageAssertionsInterface $assertions; - public function __construct(LocatorInterface|PageInterface $subject, ?TracingInterface $tracing = null) - { + /** + * Browser and platform of the run, "chromium-linux": the suffix of relative + * baseline names, as {projectName}-{platform} in @playwright/test. + */ + private readonly string $variant; + + /** + * @param string|null $outputDir Directory of the test for failure images. The browser is appended, as Playwright JS + * appends the project: each browser of a test gets its own directory. + */ + public function __construct( + LocatorInterface|PageInterface $subject, + ?TracingInterface $tracing = null, + private readonly ?string $snapshotDir = null, + ?string $outputDir = null, + ) { + $browser = self::browserName($subject); + $this->variant = implode('-', array_filter([$browser, self::platform()])); + + if (null !== $outputDir && '' !== $browser) { + $outputDir .= '-'.$browser; + } + $this->assertions = $subject instanceof LocatorInterface - ? new LocatorAssertions($subject, $tracing) - : new PageAssertions($subject, $tracing); + ? new LocatorAssertions($subject, $tracing, $outputDir, $this->variant) + : new PageAssertions($subject, $tracing, $outputDir, $this->variant); $this->assertions->withPollInterval(100); } @@ -158,6 +181,44 @@ public function toHaveURL(string $url): void $this->page(__FUNCTION__)->toHaveURL($url); } + public function toHaveScreenshot(string $name, ?ToHaveScreenshotOptions $options = null): void + { + $this->assertions->toHaveScreenshot($this->snapshotPath($name), $options); + } + + private function snapshotPath(string $name): string + { + if (str_starts_with($name, '/') || str_starts_with($name, '\\\\') || 1 === preg_match('~^[a-zA-Z]:[\\\\/]~', $name)) { + return $name; + } + + $extension = pathinfo($name, \PATHINFO_EXTENSION); + $stem = '' === $extension ? $name : substr($name, 0, -\strlen($extension) - 1); + + return sprintf('%s/%s-%s.%s', $this->snapshotDir ?? getcwd(), $stem, $this->variant, '' === $extension ? 'png' : $extension); + } + + /** + * The browser rendering the subject, read from the page under test. + * Empty when it cannot be known: a persistent context has no browser + * object, a locator built by hand has no page. + */ + private static function browserName(LocatorInterface|PageInterface $subject): string + { + try { + $page = $subject instanceof LocatorInterface ? $subject->page() : $subject; + + return $page->context()->browser()?->browserType()->value ?? ''; + } catch (RuntimeException) { + return ''; + } + } + + private static function platform(): string + { + return 'Windows' === \PHP_OS_FAMILY ? 'win32' : strtolower(\PHP_OS_FAMILY); + } + private function locator(string $matcher): LocatorAssertionsInterface { if (!$this->assertions instanceof LocatorAssertionsInterface) { diff --git a/src/Testing/ExpectDecorator.php b/src/Testing/ExpectDecorator.php index 5230ed3..7c8f3ed 100644 --- a/src/Testing/ExpectDecorator.php +++ b/src/Testing/ExpectDecorator.php @@ -14,6 +14,8 @@ namespace Playwright\Testing; +use Playwright\Assertions\Options\ToHaveScreenshotOptions; + /** * Decorator for Expect that automatically increments PHPUnit assertion count when available. * This allows the Expect class to work independently (without PHPUnit) while still @@ -152,6 +154,12 @@ public function toHaveURL(string $url): void $this->recordAssertion(1); } + public function toHaveScreenshot(string $name, ?ToHaveScreenshotOptions $options = null): void + { + $this->expect->toHaveScreenshot($name, $options); + $this->recordAssertion(1); + } + public function not(): self { $this->expect->not(); diff --git a/src/Testing/ExpectInterface.php b/src/Testing/ExpectInterface.php index 98bcdff..50aed7b 100644 --- a/src/Testing/ExpectInterface.php +++ b/src/Testing/ExpectInterface.php @@ -14,6 +14,8 @@ namespace Playwright\Testing; +use Playwright\Assertions\Options\ToHaveScreenshotOptions; + interface ExpectInterface { public function toBeAttached(): void; @@ -65,6 +67,17 @@ public function toHaveTitle(string $title): void; public function toHaveURL(string $url): void; + /** + * Compares a stable screenshot with a baseline image. + * + * A relative $name resolves in the snapshot directory of the test, suffixed + * with the browser and the platform read from the page under test: + * "card.png" in tests/CardTest.php, run in Chromium on Linux, is stored as + * tests/CardTest.php-snapshots/card-chromium-linux.png. Without extension, + * ".png" is added. Name it ".webp" for a lossless WebP baseline. + */ + public function toHaveScreenshot(string $name, ?ToHaveScreenshotOptions $options = null): void; + public function not(): self; public function withTimeout(int $timeoutMs): self; diff --git a/src/Testing/PlaywrightTestCaseTrait.php b/src/Testing/PlaywrightTestCaseTrait.php index 7de60d3..959b23c 100644 --- a/src/Testing/PlaywrightTestCaseTrait.php +++ b/src/Testing/PlaywrightTestCaseTrait.php @@ -124,7 +124,31 @@ protected function assertElementExists(string $selector): void protected function expect(LocatorInterface|PageInterface $subject): ExpectInterface { - return new ExpectDecorator(new Expect($subject, $this->context->tracing()), $this); + return new ExpectDecorator(new Expect($subject, $this->context->tracing(), $this->snapshotDirectory(), $this->testOutputDirectory()), $this); + } + + /** + * Where a failing toHaveScreenshot() leaves its expected, actual and diff + * images: one directory per test, as Playwright JS does under + * test-results/. Override to write them elsewhere. + */ + protected function testOutputDirectory(): string + { + $class = (new \ReflectionClass($this))->getShortName(); + + return getcwd().'/test-failures/'.self::sanitizeFileName($class.'-'.$this->resolveTestName()); + } + + /** + * Where toHaveScreenshot() keeps baselines: next to the test file, as in. + * + * @playwright/test. Override to store them elsewhere. + */ + protected function snapshotDirectory(): string + { + $file = (new \ReflectionClass($this))->getFileName(); + + return false === $file ? getcwd().'/snapshots' : $file.'-snapshots'; } private function resolveLogger(LoggerInterface $logger): LoggerInterface diff --git a/tests/Functional/Screenshot/ToHaveScreenshotTest.php b/tests/Functional/Screenshot/ToHaveScreenshotTest.php new file mode 100644 index 0000000..4041673 --- /dev/null +++ b/tests/Functional/Screenshot/ToHaveScreenshotTest.php @@ -0,0 +1,384 @@ +dir = sys_get_temp_dir().'/pw_to_have_screenshot_'.uniqid(); + unset($_SERVER[ScreenshotExpectation::UPDATE_ENV]); + } + + protected function tearDown(): void + { + unset($_SERVER[ScreenshotExpectation::UPDATE_ENV]); + $this->remove($this->dir); + foreach (glob(getcwd().'/test-failures/ToHaveScreenshotTest-*') ?: [] as $dir) { + $this->remove($dir); + } + + parent::tearDown(); + } + + protected function snapshotDirectory(): string + { + return $this->dir; + } + + public function testFirstRunWritesTheBaselineThenMatches(): void + { + $this->card('#818cf8'); + $options = new ToHaveScreenshotOptions(mask: [$this->page->locator('.time')]); + + try { + $this->expect($this->page)->toHaveScreenshot('page.png', $options); + $this->fail('The first run must fail after writing the baseline.'); + } catch (AssertionException $e) { + $this->assertStringContainsString('writing actual', $e->getMessage()); + } + + $baseline = $this->dir.'/page-'.$this->variant().'.png'; + $this->assertFileExists($baseline); + $this->assertSame("\x89PNG", substr((string) file_get_contents($baseline), 0, 4)); + + $this->card('#818cf8'); + $this->expect($this->page)->toHaveScreenshot('page.png', $options); + } + + public function testTimestampsAndInfiniteAnimationsAreStabilized(): void + { + $this->card('#818cf8'); + $card = $this->page->locator('#card'); + $options = new ToHaveScreenshotOptions(mask: [$this->page->locator('.time')]); + $this->record(fn () => $this->expect($card)->toHaveScreenshot('card.png', $options)); + + usleep(20_000); + $this->card('#818cf8'); + $this->expect($card)->toHaveScreenshot('card.png', $options); + } + + public function testChangedElementFailsAndKeepsTheImagesUntilItPassesAgain(): void + { + $this->card('#818cf8', 'Hello'); + $card = $this->page->locator('#card'); + $options = new ToHaveScreenshotOptions(mask: [$this->page->locator('.time')]); + $this->record(fn () => $this->expect($card)->toHaveScreenshot('card.png', $options)); + + $this->card('#818cf8', 'Hello world'); + + try { + $this->expect($card)->toHaveScreenshot('card.png', $options); + $this->fail('A changed element must fail the assertion.'); + } catch (AssertionException $e) { + $this->assertStringContainsString('Screenshot of the locator does not match', $e->getMessage()); + $this->assertStringContainsString('pixels', $e->getMessage()); + $this->assertNotNull($e->actual); + $out = dirname($e->actual).'/card'; + } + + $this->assertStringStartsWith($this->testOutputDirectory().'-chromium/', $out); + $this->assertFileExists($out.'-expected.png'); + $this->assertFileExists($out.'-actual.png'); + $this->assertFileExists($out.'-diff.png'); + + $this->card('#818cf8', 'Hello'); + $this->expect($card)->toHaveScreenshot('card.png', $options); + + $this->assertFileDoesNotExist($out.'-expected.png'); + $this->assertFileDoesNotExist($out.'-actual.png'); + $this->assertFileDoesNotExist($out.'-diff.png'); + } + + public function testMatchingAnotherBaselineKeepsTheFailureImages(): void + { + $this->card('#818cf8', 'Light'); + $card = $this->page->locator('#card'); + $options = new ToHaveScreenshotOptions(mask: [$this->page->locator('.time')]); + $this->record(fn () => $this->expect($card)->toHaveScreenshot('light/card.png', $options)); + + $this->card('#f472b6', 'Dark'); + $this->record(fn () => $this->expect($card)->toHaveScreenshot('dark/card.png', $options)); + + try { + $this->expect($card)->toHaveScreenshot('light/card.png', $options); + $this->fail('A changed element must fail the assertion.'); + } catch (AssertionException $e) { + $this->assertNotNull($e->actual); + $outputDir = dirname($e->actual); + } + + $this->expect($card)->toHaveScreenshot('dark/card.png', $options); + + foreach (['expected', 'actual', 'diff'] as $kind) { + $this->assertFileExists($outputDir.'/card-'.$kind.'.png'); + } + + $this->card('#818cf8', 'Light'); + $this->expect($card)->toHaveScreenshot('light/card.png', $options); + + $this->assertDirectoryDoesNotExist($outputDir); + } + + public function testDefaultThresholdToleratesCloseShadesOfOneHue(): void + { + $this->card('#818cf8'); + $card = $this->page->locator('#card'); + $options = new ToHaveScreenshotOptions(mask: [$this->page->locator('.time')]); + $this->record(fn () => $this->expect($card)->toHaveScreenshot('card.png', $options)); + + $this->card('#6366f1'); + $this->expect($card)->toHaveScreenshot('card.png', $options); + + $this->expectException(AssertionException::class); + $this->expect($card)->toHaveScreenshot('card.png', new ToHaveScreenshotOptions(threshold: 0.1, mask: [$this->page->locator('.time')])); + } + + public function testSsimComparatorIsAvailable(): void + { + $this->card('#818cf8'); + $card = $this->page->locator('#card'); + $options = new ToHaveScreenshotOptions(comparator: 'ssim-cie94', mask: [$this->page->locator('.time')]); + $this->record(fn () => $this->expect($card)->toHaveScreenshot('card.png', $options)); + + $this->card('#6366f1'); + + $this->expectException(AssertionException::class); + $this->expect($card)->toHaveScreenshot('card.png', $options); + } + + public function testChangedModeAcceptsTheNewScreenshot(): void + { + $this->card('#818cf8', 'Hello'); + $card = $this->page->locator('#card'); + $options = new ToHaveScreenshotOptions(mask: [$this->page->locator('.time')]); + $this->record(fn () => $this->expect($card)->toHaveScreenshot('card.png', $options)); + $before = file_get_contents($this->dir.'/card-'.$this->variant().'.png'); + + $this->card('#818cf8', 'Hello world'); + $_SERVER[ScreenshotExpectation::UPDATE_ENV] = 'changed'; + $this->expect($card)->toHaveScreenshot('card.png', $options); + unset($_SERVER[ScreenshotExpectation::UPDATE_ENV]); + + $this->assertNotSame($before, file_get_contents($this->dir.'/card-'.$this->variant().'.png')); + $this->expect($card)->toHaveScreenshot('card.png', $options); + } + + public function testWebpBaselineIsLosslessAndComparedAsWebp(): void + { + $this->card('#818cf8', 'Hello'); + $card = $this->page->locator('#card'); + $options = new ToHaveScreenshotOptions(mask: [$this->page->locator('.time')]); + $this->record(fn () => $this->expect($card)->toHaveScreenshot('card.webp', $options)); + + $baseline = (string) file_get_contents($this->dir.'/card-'.$this->variant().'.webp'); + $this->assertSame('WEBP', substr($baseline, 8, 4)); + + $this->card('#818cf8', 'Hello'); + $this->expect($card)->toHaveScreenshot('card.webp', $options); + + $this->card('#818cf8', 'Hello world'); + try { + $this->expect($card)->toHaveScreenshot('card.webp', $options); + $this->fail('A changed element must fail the assertion.'); + } catch (AssertionException $e) { + $this->assertNotNull($e->actual); + $out = dirname($e->actual).'/card'; + } + + $this->assertStringStartsWith($this->testOutputDirectory().'-chromium/', $out); + $this->assertSame('WEBP', substr((string) file_get_contents($out.'-actual.webp'), 8, 4)); + $this->assertSame("\x89PNG", substr((string) file_get_contents($out.'-diff.png'), 0, 4)); + } + + public function testFullPageCapturesBeyondTheViewport(): void + { + $this->page->setViewportSize(800, 600); + $this->page->setContent('
'); + + $this->record(fn () => $this->expect($this->page)->toHaveScreenshot('full.png', new ToHaveScreenshotOptions(fullPage: true))); + + $size = getimagesize($this->dir.'/full-'.$this->variant().'.png'); + $this->assertIsArray($size); + $this->assertSame([800, 2000], [$size[0], $size[1]]); + + $this->expect($this->page)->toHaveScreenshot('full.png', new ToHaveScreenshotOptions(fullPage: true)); + } + + public function testClipCapturesARegion(): void + { + $this->card('#818cf8'); + $clip = new ToHaveScreenshotOptions(clip: ['x' => 10, 'y' => 20, 'width' => 120, 'height' => 40]); + + $this->record(fn () => $this->expect($this->page)->toHaveScreenshot('region.png', $clip)); + + $size = getimagesize($this->dir.'/region-'.$this->variant().'.png'); + $this->assertIsArray($size); + $this->assertSame([120, 40], [$size[0], $size[1]]); + } + + public function testMaskReachesInsideAnIframe(): void + { + $frame = fn (string $time): string => sprintf( + '', + htmlspecialchars(sprintf('

Updated %s

', $time)), + ); + $options = fn (): ToHaveScreenshotOptions => new ToHaveScreenshotOptions( + mask: [$this->page->frameLocator('#preview')->locator('.time')], + ); + + $this->page->setContent($frame('10:00:00')); + $this->page->frameLocator('#preview')->locator('.time')->waitFor(); + $this->record(fn () => $this->expect($this->page->locator('#preview'))->toHaveScreenshot('frame.png', $options())); + + $this->page->setContent($frame('23:59:59')); + $this->page->frameLocator('#preview')->locator('.time')->waitFor(); + $this->expect($this->page->locator('#preview'))->toHaveScreenshot('frame.png', $options()); + + $this->expectException(AssertionException::class); + $this->expect($this->page->locator('#preview'))->toHaveScreenshot('frame.png'); + } + + public function testAllModeRewritesEvenAMatchingBaseline(): void + { + $this->card('#818cf8'); + $card = $this->page->locator('#card'); + $options = new ToHaveScreenshotOptions(mask: [$this->page->locator('.time')]); + $this->record(fn () => $this->expect($card)->toHaveScreenshot('card.png', $options)); + $baseline = $this->dir.'/card-'.$this->variant().'.png'; + file_put_contents($baseline, (string) file_get_contents($baseline)."\0"); + + $_SERVER[ScreenshotExpectation::UPDATE_ENV] = 'all'; + $this->expect($card)->toHaveScreenshot('card.png', $options); + + $this->assertStringEndsNotWith("\0", (string) file_get_contents($baseline)); + } + + public function testNoneModeNeverWritesABaseline(): void + { + $this->card('#818cf8'); + $_SERVER[ScreenshotExpectation::UPDATE_ENV] = 'none'; + + try { + $this->expect($this->page->locator('#card'))->toHaveScreenshot('card.png'); + $this->fail('A missing baseline must fail in none mode.'); + } catch (AssertionException $e) { + $this->assertStringContainsString('PLAYWRIGHT_UPDATE_SNAPSHOTS=missing', $e->getMessage()); + } + + $this->assertFileDoesNotExist($this->dir.'/card-'.$this->variant().'.png'); + } + + public function testEmptyBaselineFailsThroughTheBridge(): void + { + $this->card('#818cf8'); + mkdir($this->dir, 0777, true); + file_put_contents($this->dir.'/card-'.$this->variant().'.png', ''); + + $this->expectException(AssertionException::class); + $this->expectExceptionMessage('is empty'); + + $this->expect($this->page->locator('#card'))->toHaveScreenshot('card.png'); + } + + public function testFullPageIsRejectedOnAnElement(): void + { + $this->card('#818cf8'); + + $this->expectException(\InvalidArgumentException::class); + $this->expectExceptionMessage('The fullPage and clip options apply to a page screenshot'); + + $this->expect($this->page->locator('#card'))->toHaveScreenshot('card.png', new ToHaveScreenshotOptions(fullPage: true)); + } + + public function testCannotBeNegated(): void + { + $this->card('#818cf8'); + + $this->expectException(\LogicException::class); + $this->expectExceptionMessage('toHaveScreenshot() cannot be negated.'); + + $this->expect($this->page)->not()->toHaveScreenshot('page.png'); + } + + private function card(string $color, string $title = 'Hello'): void + { + $this->page->setContent(sprintf(<<<'HTML' + +
+

%s

+

%s

+
+
+ + + HTML, $color, $title, microtime())); + } + + /** + * Runs a first assertion that records the baseline, which fails by design. + */ + private function record(callable $assertion): void + { + try { + $assertion(); + } catch (AssertionException $e) { + $this->assertStringContainsString('writing actual', $e->getMessage()); + } + } + + /** + * The functional suite runs Chromium. + */ + private function variant(): string + { + return 'chromium-'.('Windows' === \PHP_OS_FAMILY ? 'win32' : strtolower(\PHP_OS_FAMILY)); + } + + private function remove(string $dir): void + { + if (!is_dir($dir)) { + return; + } + + foreach (glob($dir.'/*') ?: [] as $file) { + is_dir($file) ? $this->remove($file) : unlink($file); + } + rmdir($dir); + } +} diff --git a/tests/Unit/Assertions/Internal/ScreenshotExpectationTest.php b/tests/Unit/Assertions/Internal/ScreenshotExpectationTest.php new file mode 100644 index 0000000..5410b65 --- /dev/null +++ b/tests/Unit/Assertions/Internal/ScreenshotExpectationTest.php @@ -0,0 +1,569 @@ +}> */ + private array $calls = []; + + protected function setUp(): void + { + $this->dir = sys_get_temp_dir().'/pw_screenshot_expectation_'.uniqid(); + mkdir($this->dir); + $this->calls = []; + } + + protected function tearDown(): void + { + $iterator = new \RecursiveIteratorIterator( + new \RecursiveDirectoryIterator($this->dir, \FilesystemIterator::SKIP_DOTS), + \RecursiveIteratorIterator::CHILD_FIRST, + ); + foreach ($iterator as $file) { + $file->isDir() ? rmdir($file->getPathname()) : unlink($file->getPathname()); + } + rmdir($this->dir); + } + + public function testMissingBaselineIsWrittenAndFailsByDefault(): void + { + $path = $this->dir.'/snapshots/card.png'; + + try { + $this->expectation(new ScreenshotComparison(actual: 'stable'))->assert($path, new ToHaveScreenshotOptions(), 5000); + $this->fail('A missing baseline must fail the assertion.'); + } catch (AssertionException $e) { + $this->assertStringContainsString("doesn't exist at {$path}, writing actual", $e->getMessage()); + } + + $this->assertSame('stable', file_get_contents($path)); + $this->assertNull($this->calls[0]['expected']); + } + + #[DataProvider('provideUpdatingModes')] + public function testMissingBaselineIsWrittenAndPassesWhenUpdating(string $mode): void + { + $path = $this->dir.'/card.png'; + + $this->expectation(new ScreenshotComparison(actual: 'stable'), $mode)->assert($path, new ToHaveScreenshotOptions(), 5000); + + $this->assertSame('stable', file_get_contents($path)); + } + + /** + * @return iterable + */ + public static function provideUpdatingModes(): iterable + { + yield 'changed' => ['changed']; + yield 'all' => ['all']; + } + + public function testMissingBaselineIsNotWrittenInNoneMode(): void + { + $path = $this->dir.'/card.png'; + + $this->expectException(AssertionException::class); + $this->expectExceptionMessage('Run with PLAYWRIGHT_UPDATE_SNAPSHOTS=missing to write it.'); + + try { + $this->expectation(new ScreenshotComparison(actual: 'stable'), 'none')->assert($path, new ToHaveScreenshotOptions(), 5000); + } finally { + $this->assertFileDoesNotExist($path); + } + } + + public function testUnstablePageFailsWithoutWritingABaseline(): void + { + $path = $this->dir.'/card.png'; + $unstable = new ScreenshotComparison(actual: 'last', errorMessage: 'Failed to take two consecutive stable screenshots.', timedOut: true); + + $this->expectException(AssertionException::class); + $this->expectExceptionMessage('Failed to take a stable screenshot of the page: Failed to take two consecutive stable screenshots.'); + + try { + $this->expectation($unstable)->assert($path, new ToHaveScreenshotOptions(), 5000); + } finally { + $this->assertFileDoesNotExist($path); + } + } + + public function testMatchingScreenshotPassesAndSendsTheBaseline(): void + { + $path = $this->baseline('baseline'); + + $this->expectation(new ScreenshotComparison(actual: null))->assert($path, new ToHaveScreenshotOptions(threshold: 0.1), 1234); + + $this->assertSame('baseline', $this->calls[0]['expected']); + $this->assertSame(0.1, $this->calls[0]['options']['threshold']); + $this->assertSame(1234, $this->calls[0]['options']['timeout']); + $this->assertSame('png', $this->calls[0]['options']['type']); + } + + public function testOptionTimeoutWinsOverTheAssertionTimeout(): void + { + $path = $this->baseline('baseline'); + + $this->expectation(new ScreenshotComparison(actual: null))->assert($path, new ToHaveScreenshotOptions(timeoutMs: 42), 5000); + + $this->assertSame(42, $this->calls[0]['options']['timeout']); + } + + public function testMismatchKeepsExpectedActualAndDiff(): void + { + $path = $this->baseline('baseline'); + $mismatch = new ScreenshotComparison(actual: 'actual', diff: 'diff', errorMessage: '120 pixels (ratio 0.02 of all image pixels) are different.'); + + try { + $this->expectation($mismatch)->assert($path, new ToHaveScreenshotOptions(), 5000); + $this->fail('A mismatch must fail the assertion.'); + } catch (AssertionException $e) { + $this->assertStringContainsString("Screenshot of the page does not match {$path}.", $e->getMessage()); + $this->assertStringContainsString('120 pixels (ratio 0.02 of all image pixels) are different.', $e->getMessage()); + $this->assertStringContainsString('PLAYWRIGHT_UPDATE_SNAPSHOTS=changed', $e->getMessage()); + $this->assertSame($this->artifact('card-actual.png'), $e->actual); + } + + $this->assertSame('baseline', file_get_contents($this->artifact('card-expected.png'))); + $this->assertSame('actual', file_get_contents($this->artifact('card-actual.png'))); + $this->assertSame('diff', file_get_contents($this->artifact('card-diff.png'))); + $this->assertSame('baseline', file_get_contents($path)); + } + + public function testMatchRemovesTheImagesOfAnEarlierFailure(): void + { + $path = $this->baseline('baseline'); + $this->staleArtifacts(); + + $this->expectation(new ScreenshotComparison(actual: null))->assert($path, new ToHaveScreenshotOptions(), 5000); + + $this->assertDirectoryDoesNotExist($this->dir.'/out'); + } + + /** + * @param 'changed'|'all' $mode + */ + #[DataProvider('provideUpdatingModes')] + public function testRecordingRemovesTheImagesOfAnEarlierFailure(string $mode): void + { + $path = $this->baseline('baseline'); + $this->staleArtifacts(); + + $this->expectation(new ScreenshotComparison(actual: 'actual', errorMessage: 'changed' === $mode ? 'different' : null), $mode) + ->assert($path, new ToHaveScreenshotOptions(), 5000); + + $this->assertDirectoryDoesNotExist($this->dir.'/out'); + } + + public function testFailureKeepsTheImagesOfOtherBaselines(): void + { + $path = $this->baseline('baseline'); + mkdir($this->dir.'/out'); + file_put_contents($this->dir.'/out/other-actual.png', 'other'); + + $this->expectation(new ScreenshotComparison(actual: null))->assert($path, new ToHaveScreenshotOptions(), 5000); + + $this->assertFileExists($this->dir.'/out/other-actual.png'); + } + + #[DataProvider('provideCollidingBaselineNames')] + public function testArtifactsBelongOnlyToTheirBaseline(string $firstName, string $secondName, string $cleanupMode): void + { + $paths = [$this->dir.'/'.$firstName, $this->dir.'/'.$secondName]; + $directories = []; + + foreach ($paths as $index => $path) { + if (!is_dir(dirname($path))) { + mkdir(dirname($path), 0777, true); + } + file_put_contents($path, 'baseline '.$index); + + try { + $this->expectation(new ScreenshotComparison(actual: 'actual '.$index, diff: 'diff '.$index, errorMessage: 'different')) + ->assert($path, new ToHaveScreenshotOptions(), 5000); + $this->fail('A mismatch must fail the assertion.'); + } catch (AssertionException $e) { + $this->assertNotNull($e->actual); + $directories[] = dirname($e->actual); + } + } + + $this->assertNotSame($directories[0], $directories[1]); + foreach ($paths as $index => $path) { + $extension = pathinfo($path, \PATHINFO_EXTENSION); + $this->assertSame('baseline '.$index, file_get_contents($directories[$index].'/card-expected.'.$extension)); + $this->assertSame('actual '.$index, file_get_contents($directories[$index].'/card-actual.'.$extension)); + $this->assertSame('diff '.$index, file_get_contents($directories[$index].'/card-diff.png')); + } + + $result = match ($cleanupMode) { + 'changed' => new ScreenshotComparison(actual: 'updated', errorMessage: 'different'), + 'all' => new ScreenshotComparison(actual: 'updated'), + default => new ScreenshotComparison(actual: null), + }; + $this->expectation($result, $cleanupMode)->assert($paths[0], new ToHaveScreenshotOptions(), 5000); + + $this->assertDirectoryDoesNotExist($directories[0]); + $this->assertSame('diff 1', file_get_contents($directories[1].'/card-diff.png')); + + $this->expectation(new ScreenshotComparison(actual: null))->assert($paths[1], new ToHaveScreenshotOptions(), 5000); + + $this->assertDirectoryDoesNotExist($this->dir.'/out'); + } + + /** + * @return iterable + */ + public static function provideCollidingBaselineNames(): iterable + { + foreach (['none', 'changed', 'all'] as $mode) { + yield 'different directories, '.$mode => ['a/card.png', 'b/card.png', $mode]; + yield 'different formats, '.$mode => ['card.png', 'card.webp', $mode]; + } + } + + public function testMatchingTheSameBaselineThroughAnotherPathCleansItsArtifacts(): void + { + $this->baseline('baseline'); + $this->staleArtifacts(); + + $this->expectation(new ScreenshotComparison(actual: null)) + ->assert($this->dir.'/./card.png', new ToHaveScreenshotOptions(), 5000); + + $this->assertDirectoryDoesNotExist($this->dir.'/out'); + } + + public function testFailureImagesDropTheVariantOfTheBaselineName(): void + { + $path = $this->dir.'/card-chromium-linux.webp'; + file_put_contents($path, 'baseline'); + $mismatch = new ScreenshotComparison(actual: 'actual', diff: 'diff', errorMessage: 'different'); + + try { + $this->expectation($mismatch, variant: 'chromium-linux')->assert($path, new ToHaveScreenshotOptions(), 5000); + $this->fail('A mismatch must fail the assertion.'); + } catch (AssertionException) { + } + + $this->assertSame(['card-actual.webp', 'card-diff.png', 'card-expected.webp'], array_map('basename', glob($this->dir.'/out/*/*') ?: [])); + + $this->expectation(new ScreenshotComparison(actual: null), variant: 'chromium-linux')->assert($path, new ToHaveScreenshotOptions(), 5000); + + $this->assertDirectoryDoesNotExist($this->dir.'/out'); + } + + public function testFailureImagesKeepANameThatDoesNotEndWithTheVariant(): void + { + $path = $this->baseline('baseline'); + + try { + $this->expectation(new ScreenshotComparison(actual: 'actual', errorMessage: 'different'), variant: 'chromium-linux') + ->assert($path, new ToHaveScreenshotOptions(), 5000); + } catch (AssertionException) { + } + + $this->assertFileExists($this->artifact('card-actual.png')); + } + + public function testMismatchUsesTheCustomMessage(): void + { + $path = $this->baseline('baseline'); + + $this->expectException(AssertionException::class); + $this->expectExceptionMessage('The card changed.'); + + $this->expectation(new ScreenshotComparison(actual: 'actual', errorMessage: 'different')) + ->assert($path, new ToHaveScreenshotOptions(message: 'The card changed.'), 5000); + } + + public function testChangedModeRewritesAMismatchingBaseline(): void + { + $path = $this->baseline('baseline'); + + $this->expectation(new ScreenshotComparison(actual: 'actual', errorMessage: 'different'), 'changed')->assert($path, new ToHaveScreenshotOptions(), 5000); + + $this->assertSame('actual', file_get_contents($path)); + } + + public function testChangedModeKeepsAMatchingBaseline(): void + { + $path = $this->baseline('baseline'); + + $this->expectation(new ScreenshotComparison(actual: null), 'changed')->assert($path, new ToHaveScreenshotOptions(), 5000); + + $this->assertSame('baseline', file_get_contents($path)); + } + + public function testAllModeRecordsWithoutComparing(): void + { + $path = $this->baseline('baseline'); + + $this->expectation(new ScreenshotComparison(actual: 'stable'), 'all')->assert($path, new ToHaveScreenshotOptions(), 5000); + + $this->assertSame('stable', file_get_contents($path)); + $this->assertNull($this->calls[0]['expected']); + } + + public function testAllModeNeverRecordsAnUnstableScreenshot(): void + { + $path = $this->baseline('baseline'); + + $this->expectException(AssertionException::class); + $this->expectExceptionMessage('Failed to take a stable screenshot'); + + try { + $this->expectation(new ScreenshotComparison(actual: 'moving', errorMessage: 'unstable', timedOut: true), 'all') + ->assert($path, new ToHaveScreenshotOptions(), 5000); + } finally { + $this->assertSame('baseline', file_get_contents($path)); + } + } + + public function testMismatchFailsInNoneMode(): void + { + $path = $this->baseline('baseline'); + + $this->expectException(AssertionException::class); + + try { + $this->expectation(new ScreenshotComparison(actual: 'actual', errorMessage: 'different'), 'none')->assert($path, new ToHaveScreenshotOptions(), 5000); + } finally { + $this->assertSame('baseline', file_get_contents($path)); + } + } + + public function testTimedOutMismatchNeverRewritesTheBaseline(): void + { + $path = $this->baseline('baseline'); + + $this->expectException(AssertionException::class); + + try { + $this->expectation(new ScreenshotComparison(actual: 'moving', errorMessage: 'unstable', timedOut: true), 'changed') + ->assert($path, new ToHaveScreenshotOptions(), 5000); + } finally { + $this->assertSame('baseline', file_get_contents($path)); + } + } + + public function testWebpBaselineRequestsWebpAndKeepsWebpArtifacts(): void + { + $path = $this->dir.'/card.webp'; + file_put_contents($path, 'baseline'); + + try { + $this->expectation(new ScreenshotComparison(actual: 'actual', diff: 'diff', errorMessage: 'different'))->assert($path, new ToHaveScreenshotOptions(), 5000); + $this->fail('A mismatch must fail the assertion.'); + } catch (AssertionException) { + } + + $this->assertSame('webp', $this->calls[0]['options']['type']); + $this->assertFileExists($this->artifact('card-expected.webp')); + $this->assertFileExists($this->artifact('card-actual.webp')); + $this->assertFileExists($this->artifact('card-diff.png')); + } + + public function testJpegBaselineIsRejected(): void + { + $this->expectException(\InvalidArgumentException::class); + $this->expectExceptionMessage('must be a .png or .webp file.'); + + $this->expectation(new ScreenshotComparison(actual: 'stable'))->assert($this->dir.'/card.jpg', new ToHaveScreenshotOptions(), 5000); + } + + public function testUnstablePageWithABaselineKeepsTheLastScreenshot(): void + { + $path = $this->baseline('baseline'); + + try { + $this->expectation(new ScreenshotComparison(actual: 'moving', errorMessage: 'Failed to take two consecutive stable screenshots.', timedOut: true)) + ->assert($path, new ToHaveScreenshotOptions(), 5000); + $this->fail('An unstable page must fail the assertion.'); + } catch (AssertionException $e) { + $this->assertStringContainsString('Failed to take two consecutive stable screenshots.', $e->getMessage()); + } + + $this->assertSame('baseline', file_get_contents($path)); + $this->assertSame('baseline', file_get_contents($this->artifact('card-expected.png'))); + $this->assertSame('moving', file_get_contents($this->artifact('card-actual.png'))); + } + + public function testUnreadableBaselineIsReported(): void + { + if ('Windows' === \PHP_OS_FAMILY || !function_exists('posix_geteuid')) { + $this->markTestSkipped('Needs POSIX file permissions.'); + } + if (0 === posix_geteuid()) { + $this->markTestSkipped('Root reads files whatever their permissions.'); + } + + $path = $this->baseline('baseline'); + chmod($path, 0o000); + + $this->expectException(\RuntimeException::class); + $this->expectExceptionMessage(sprintf('Unable to read the snapshot "%s".', $path)); + + try { + $this->expectation(new ScreenshotComparison(actual: null))->assert($path, new ToHaveScreenshotOptions(), 5000); + } finally { + chmod($path, 0o644); + } + } + + public function testUncreatableOutputDirectoryIsReported(): void + { + $path = $this->baseline('baseline'); + file_put_contents($this->dir.'/file', 'not a directory'); + + $expectation = new ScreenshotExpectation( + static fn (): ScreenshotComparison => new ScreenshotComparison(actual: 'actual', errorMessage: 'different'), + 'page', + 'none', + $this->dir.'/file/out', + ); + + $this->expectException(\RuntimeException::class); + $this->expectExceptionMessage(sprintf('Directory "%s/file/out/', $this->dir)); + + $expectation->assert($path, new ToHaveScreenshotOptions(), 5000); + } + + public function testUnwritableBaselineIsReported(): void + { + $path = $this->dir.'/card.png'; + mkdir($path); + + $this->expectException(\RuntimeException::class); + $this->expectExceptionMessage(sprintf('Unable to write "%s".', $path)); + + $this->expectation(new ScreenshotComparison(actual: 'stable'), 'changed')->assert($path, new ToHaveScreenshotOptions(), 5000); + } + + /** + * @param 'none'|'missing' $mode + */ + #[DataProvider('provideComparingModes')] + public function testEmptyBaselineFailsWithoutComparing(string $mode): void + { + $path = $this->baseline(''); + + try { + $this->expectation(new ScreenshotComparison(actual: null), $mode)->assert($path, new ToHaveScreenshotOptions(), 5000); + $this->fail('An empty baseline must fail the assertion.'); + } catch (AssertionException $e) { + $this->assertSame(sprintf('The snapshot at %s is empty. Run with PLAYWRIGHT_UPDATE_SNAPSHOTS=changed to record it again.', $path), $e->getMessage()); + } + + $this->assertSame([], $this->calls); + $this->assertSame('', file_get_contents($path)); + } + + /** + * @return iterable + */ + public static function provideComparingModes(): iterable + { + yield 'none' => ['none']; + yield 'missing' => ['missing']; + } + + #[DataProvider('provideUpdatingModes')] + public function testEmptyBaselineIsRecordedAgainWhenUpdating(string $mode): void + { + $path = $this->baseline(''); + + $this->expectation(new ScreenshotComparison(actual: 'stable'), $mode)->assert($path, new ToHaveScreenshotOptions(), 5000); + + $this->assertSame('stable', file_get_contents($path)); + $this->assertNull($this->calls[0]['expected']); + } + + public function testFailureReplacesTheImagesOfThePreviousFailure(): void + { + $path = $this->baseline('baseline'); + + foreach ([new ScreenshotComparison(actual: 'first', diff: 'diff', errorMessage: 'different'), new ScreenshotComparison(actual: 'moving', errorMessage: 'unstable', timedOut: true)] as $result) { + try { + $this->expectation($result)->assert($path, new ToHaveScreenshotOptions(), 5000); + $this->fail('A mismatch must fail the assertion.'); + } catch (AssertionException) { + } + } + + $this->assertSame(['card-actual.png', 'card-expected.png'], array_map('basename', glob($this->dir.'/out/*/*') ?: [])); + $this->assertSame('moving', file_get_contents($this->artifact('card-actual.png'))); + } + + public function testUnknownUpdateModeIsRejected(): void + { + $this->expectException(\InvalidArgumentException::class); + $this->expectExceptionMessage('PLAYWRIGHT_UPDATE_SNAPSHOTS must be one of "none", "missing", "changed", "all", "yes" given.'); + + $this->expectation(new ScreenshotComparison(actual: 'stable'), 'yes')->assert($this->dir.'/card.png', new ToHaveScreenshotOptions(), 5000); + } + + private function artifact(string $name): string + { + $files = glob($this->dir.'/out/*/'.$name) ?: []; + $this->assertCount(1, $files); + + return $files[0]; + } + + private function staleArtifacts(): void + { + try { + $this->expectation(new ScreenshotComparison(actual: 'stale', diff: 'stale', errorMessage: 'different')) + ->assert($this->dir.'/card.png', new ToHaveScreenshotOptions(), 5000); + $this->fail('A mismatch must fail the assertion.'); + } catch (AssertionException) { + } + } + + private function baseline(string $contents): string + { + $path = $this->dir.'/card.png'; + file_put_contents($path, $contents); + + return $path; + } + + private function expectation(ScreenshotComparison $result, ?string $mode = null, ?string $variant = null): ScreenshotExpectation + { + return new ScreenshotExpectation( + function (?string $expected, array $options) use ($result): ScreenshotComparison { + $this->calls[] = ['expected' => $expected, 'options' => $options]; + + return $result; + }, + 'page', + $mode, + $this->dir.'/out', + $variant, + ); + } +} diff --git a/tests/Unit/Assertions/LocatorAssertionsTest.php b/tests/Unit/Assertions/LocatorAssertionsTest.php index 56edce2..7acac2c 100644 --- a/tests/Unit/Assertions/LocatorAssertionsTest.php +++ b/tests/Unit/Assertions/LocatorAssertionsTest.php @@ -15,11 +15,17 @@ namespace Playwright\Tests\Unit\Assertions; use PHPUnit\Framework\Attributes\CoversClass; +use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\TestCase; use Playwright\Assertions\AssertionOptions; use Playwright\Assertions\Failure\AssertionException; +use Playwright\Assertions\Internal\ScreenshotExpectation; use Playwright\Assertions\LocatorAssertions; +use Playwright\Assertions\Options\ToHaveScreenshotOptions; +use Playwright\Locator\Locator; use Playwright\Locator\LocatorInterface; +use Playwright\Tracing\TracingInterface; +use Playwright\Transport\TransportInterface; #[CoversClass(LocatorAssertions::class)] final class LocatorAssertionsTest extends TestCase @@ -364,4 +370,129 @@ public function testToBeInViewportRejectsANegativeRatio(): void (new LocatorAssertions($this->createMock(LocatorInterface::class))) ->toBeInViewport(new AssertionOptions(ratio: -0.1)); } + + /** @var list */ + private array $dirs = []; + + public function testToHaveScreenshotNeedsAPlaywrightLocator(): void + { + $this->expectException(\LogicException::class); + $this->expectExceptionMessage('toHaveScreenshot() needs a Playwright\Locator\Locator created by Playwright PHP'); + + (new LocatorAssertions($this->createStub(LocatorInterface::class)))->toHaveScreenshot('/tmp/card.png'); + } + + /** + * @param array $arguments + */ + #[DataProvider('providePageOnlyOptions')] + public function testToHaveScreenshotRejectsPageOnlyOptions(array $arguments): void + { + $transport = $this->createMock(TransportInterface::class); + $transport->expects($this->never())->method('send'); + + $this->expectException(\InvalidArgumentException::class); + $this->expectExceptionMessage('The fullPage and clip options apply to a page screenshot'); + + (new LocatorAssertions(new Locator($transport, 'page1', '#card')))->toHaveScreenshot('/tmp/card.png', new ToHaveScreenshotOptions(...$arguments)); + } + + /** + * @return iterable}> + */ + public static function providePageOnlyOptions(): iterable + { + yield 'fullPage' => [['fullPage' => true]]; + yield 'clip' => [['clip' => ['x' => 0, 'y' => 0, 'width' => 10, 'height' => 10]]]; + } + + public function testToHaveScreenshotComparesWithTheBaselineInsideATracingGroup(): void + { + $path = $this->snapshotDir().'/card.png'; + file_put_contents($path, 'baseline'); + + $transport = $this->createMock(TransportInterface::class); + $transport->expects($this->once()) + ->method('send') + ->with($this->callback(static fn (array $payload): bool => 'locator.expectScreenshot' === $payload['action'] + && base64_encode('baseline') === $payload['expected'] + && 1234 === $payload['options']['timeout'] + && 'png' === $payload['options']['type'])) + ->willReturn(['errorMessage' => null]); + + $tracing = $this->createMock(TracingInterface::class); + $tracing->expects($this->once())->method('group')->with('expect(#card).toHaveScreenshot'); + $tracing->expects($this->once())->method('groupEnd'); + + $assertions = new LocatorAssertions(new Locator($transport, 'page1', '#card'), $tracing); + + $this->assertSame($assertions, $assertions->withTimeout(1234)->toHaveScreenshot($path)); + } + + public function testToHaveScreenshotWritesFailureImagesInTheGivenDirectory(): void + { + $dir = $this->snapshotDir(); + $path = $dir.'/card.png'; + file_put_contents($path, 'baseline'); + $_SERVER[ScreenshotExpectation::UPDATE_ENV] = 'none'; + + $transport = $this->createStub(TransportInterface::class); + $transport->method('send')->willReturn(['actual' => base64_encode('actual'), 'diff' => base64_encode('diff'), 'errorMessage' => 'different']); + + try { + (new LocatorAssertions(new Locator($transport, 'page1', '#card'), null, $dir.'/out'))->toHaveScreenshot($path); + $this->fail('A mismatch must fail the assertion.'); + } catch (AssertionException $e) { + $this->assertNotNull($e->actual); + $outputDir = dirname($e->actual); + } + + $this->assertStringStartsWith($dir.'/out/', $outputDir); + $this->assertSame('actual', file_get_contents($outputDir.'/card-actual.png')); + $this->assertSame('diff', file_get_contents($outputDir.'/card-diff.png')); + foreach (glob($outputDir.'/*') ?: [] as $file) { + unlink($file); + } + rmdir($outputDir); + rmdir($dir.'/out'); + } + + public function testToHaveScreenshotCannotBeNegatedAndResetsTheNegation(): void + { + $path = $this->snapshotDir().'/card.png'; + file_put_contents($path, 'baseline'); + + $transport = $this->createMock(TransportInterface::class); + $transport->expects($this->once())->method('send')->willReturn(['errorMessage' => null]); + $assertions = new LocatorAssertions(new Locator($transport, 'page1', '#card')); + + try { + $assertions->not()->toHaveScreenshot($path); + $this->fail('A negated screenshot assertion must throw.'); + } catch (\LogicException $e) { + $this->assertSame('toHaveScreenshot() cannot be negated.', $e->getMessage()); + } + + $this->assertSame($assertions, $assertions->toHaveScreenshot($path)); + } + + private function snapshotDir(): string + { + $dir = sys_get_temp_dir().'/pw-assertions-'.bin2hex(random_bytes(6)); + mkdir($dir); + $this->dirs[] = $dir; + + return $dir; + } + + protected function tearDown(): void + { + unset($_SERVER[ScreenshotExpectation::UPDATE_ENV]); + foreach ($this->dirs as $dir) { + foreach (glob($dir.'/*') ?: [] as $file) { + unlink($file); + } + rmdir($dir); + } + } } diff --git a/tests/Unit/Assertions/Options/ToHaveScreenshotOptionsTest.php b/tests/Unit/Assertions/Options/ToHaveScreenshotOptionsTest.php new file mode 100644 index 0000000..e1b50a0 --- /dev/null +++ b/tests/Unit/Assertions/Options/ToHaveScreenshotOptionsTest.php @@ -0,0 +1,157 @@ +assertSame([ + 'threshold' => 0.2, + 'comparator' => 'pixelmatch', + 'maskColor' => '#FF00FF', + 'fullPage' => false, + 'omitBackground' => false, + 'animations' => 'disabled', + 'caret' => 'hide', + 'scale' => 'css', + ], (new ToHaveScreenshotOptions())->toArray()); + } + + public function testToleranceHasNoDefault(): void + { + $options = new ToHaveScreenshotOptions(); + + $this->assertNull($options->maxDiffPixels); + $this->assertNull($options->maxDiffPixelRatio); + $this->assertNull($options->timeoutMs); + } + + public function testSendsTheValuesSet(): void + { + $options = new ToHaveScreenshotOptions( + threshold: 0.1, + maxDiffPixels: 10, + maxDiffPixelRatio: 0.01, + comparator: ToHaveScreenshotOptions::COMPARATOR_SSIM_CIE94, + style: '.ad { display: none }', + fullPage: true, + clip: ['x' => 0, 'y' => 0, 'width' => 320, 'height' => 200], + omitBackground: true, + animations: 'allow', + caret: 'initial', + scale: 'device', + ); + + $this->assertSame([ + 'threshold' => 0.1, + 'maxDiffPixels' => 10, + 'maxDiffPixelRatio' => 0.01, + 'comparator' => 'ssim-cie94', + 'maskColor' => '#FF00FF', + 'style' => '.ad { display: none }', + 'fullPage' => true, + 'clip' => ['x' => 0, 'y' => 0, 'width' => 320, 'height' => 200], + 'omitBackground' => true, + 'animations' => 'allow', + 'caret' => 'initial', + 'scale' => 'device', + ], $options->toArray()); + } + + public function testTimeoutAndMessageStayOnThePhpSide(): void + { + $options = (new ToHaveScreenshotOptions(timeoutMs: 1000, message: 'The card changed.'))->toArray(); + + $this->assertArrayNotHasKey('timeoutMs', $options); + $this->assertArrayNotHasKey('timeout', $options); + $this->assertArrayNotHasKey('message', $options); + } + + public function testSerializesMaskLocatorsAsSelectors(): void + { + $locator = $this->createStub(LocatorInterface::class); + $locator->method('getSelector')->willReturn('.timestamp'); + + $options = new ToHaveScreenshotOptions(mask: [$locator], maskColor: 'black'); + + $this->assertSame('black', $options->toArray()['maskColor']); + $this->assertSame([['selector' => '.timestamp']], $options->toArray()['mask']); + } + + public function testMaskLocatorsInAFrameKeepTheirFrame(): void + { + $locator = new Locator($this->createStub(TransportInterface::class), 'page1', '.date', 'iframe#preview'); + + $options = new ToHaveScreenshotOptions(mask: [$locator]); + + $this->assertSame([['selector' => '.date', 'frameSelector' => 'iframe#preview']], $options->toArray()['mask']); + } + + /** + * @param array $arguments + */ + #[DataProvider('provideInvalidOptions')] + public function testRejectsInvalidOptions(array $arguments, string $message): void + { + $this->expectException(\InvalidArgumentException::class); + $this->expectExceptionMessage($message); + + new ToHaveScreenshotOptions(...$arguments); + } + + /** + * @return iterable, string}> + */ + public static function provideInvalidOptions(): iterable + { + foreach (['NaN' => \NAN, 'positive infinity' => \INF, 'negative infinity' => -\INF] as $label => $value) { + foreach (['threshold', 'maxDiffPixelRatio'] as $option) { + yield $option.' '.$label => [[$option => $value], sprintf('The %s option must be between 0 and 1', $option)]; + } + foreach (['x', 'y', 'width', 'height'] as $coordinate) { + $clip = ['x' => 0, 'y' => 0, 'width' => 10, 'height' => 10]; + $clip[$coordinate] = $value; + + yield 'clip '.$coordinate.' '.$label => [['clip' => $clip], sprintf('The clip option needs a finite "%s" value.', $coordinate)]; + } + } + + yield 'threshold above 1' => [['threshold' => 2.0], 'The threshold option must be between 0 and 1, 2 given.']; + yield 'negative threshold' => [['threshold' => -0.1], 'The threshold option must be between 0 and 1, -0.1 given.']; + yield 'ratio above 1' => [['maxDiffPixelRatio' => 1.5], 'The maxDiffPixelRatio option must be between 0 and 1, 1.5 given.']; + yield 'negative pixels' => [['maxDiffPixels' => -1], 'The maxDiffPixels option must be 0 or more, -1 given.']; + yield 'negative timeout' => [['timeoutMs' => -1], 'The timeoutMs option must be 0 or more, -1 given.']; + yield 'unknown comparator' => [['comparator' => 'ssim'], 'The comparator option must be one of "pixelmatch", "ssim-cie94", "ssim" given.']; + yield 'unknown animations' => [['animations' => 'none'], 'The animations option must be one of "disabled", "allow", "none" given.']; + yield 'unknown caret' => [['caret' => 'hidden'], 'The caret option must be one of "hide", "initial", "hidden" given.']; + yield 'unknown scale' => [['scale' => 'px'], 'The scale option must be one of "css", "device", "px" given.']; + yield 'empty mask color' => [['maskColor' => ' '], 'The maskColor option must be a CSS color, an empty string given.']; + yield 'selector in mask' => [['mask' => ['.date']], 'The mask option expects Playwright\Locator\LocatorInterface instances, string given at index 0.']; + yield 'clip without height' => [['clip' => ['x' => 0, 'y' => 0, 'width' => 10]], 'The clip option needs numeric "x", "y", "width" and "height" keys, "height" is missing or not a number.']; + yield 'clip with a string' => [['clip' => ['x' => '0', 'y' => 0, 'width' => 10, 'height' => 10]], '"x" is missing or not a number.']; + yield 'empty clip' => [['clip' => ['x' => 0, 'y' => 0, 'width' => 0, 'height' => 10]], 'The clip option needs a width and a height greater than 0.']; + yield 'clip with an extra key' => [['clip' => ['x' => 0, 'y' => 0, 'width' => 10, 'height' => 10, 'scale' => 2]], 'The clip option only accepts "x", "y", "width" and "height" keys, "x", "y", "width", "height", "scale" given.']; + } +} diff --git a/tests/Unit/Assertions/PageAssertionsTest.php b/tests/Unit/Assertions/PageAssertionsTest.php index 1f59720..f09e5e5 100644 --- a/tests/Unit/Assertions/PageAssertionsTest.php +++ b/tests/Unit/Assertions/PageAssertionsTest.php @@ -18,10 +18,16 @@ use PHPUnit\Framework\TestCase; use Playwright\Assertions\AssertionOptions; use Playwright\Assertions\Failure\AssertionException; +use Playwright\Assertions\Internal\ScreenshotExpectation; +use Playwright\Assertions\Options\ToHaveScreenshotOptions; use Playwright\Assertions\PageAssertions; +use Playwright\Browser\BrowserContextInterface; +use Playwright\Configuration\PlaywrightConfig; use Playwright\Locator\LocatorInterface; +use Playwright\Page\Page; use Playwright\Page\PageInterface; use Playwright\Tracing\TracingInterface; +use Playwright\Transport\TransportInterface; #[CoversClass(PageAssertions::class)] final class PageAssertionsTest extends TestCase @@ -128,4 +134,90 @@ private function pageSnapshotting(string $snapshot): PageInterface return $page; } + + /** @var list */ + private array $dirs = []; + + public function testToHaveScreenshotNeedsAPlaywrightPage(): void + { + $this->expectException(\LogicException::class); + $this->expectExceptionMessage('toHaveScreenshot() needs a Playwright\Page\Page created by Playwright PHP'); + + (new PageAssertions($this->createStub(PageInterface::class)))->toHaveScreenshot('/tmp/page.png'); + } + + public function testToHaveScreenshotSendsPageOptionsInsideATracingGroup(): void + { + $path = $this->snapshotDir().'/page.png'; + file_put_contents($path, 'baseline'); + + $transport = $this->createMock(TransportInterface::class); + $transport->expects($this->once()) + ->method('send') + ->with($this->callback(static fn (array $payload): bool => 'page.expectScreenshot' === $payload['action'] + && true === $payload['options']['fullPage'] + && ['x' => 0, 'y' => 0, 'width' => 320, 'height' => 200] === $payload['options']['clip'])) + ->willReturn(['errorMessage' => null]); + + $tracing = $this->createMock(TracingInterface::class); + $tracing->expects($this->once())->method('group')->with('expect(page).toHaveScreenshot'); + $tracing->expects($this->once())->method('groupEnd'); + + $page = new Page($transport, $this->createStub(BrowserContextInterface::class), 'page-id', new PlaywrightConfig()); + $assertions = new PageAssertions($page, $tracing); + + $this->assertSame($assertions, $assertions->toHaveScreenshot($path, new ToHaveScreenshotOptions( + fullPage: true, + clip: ['x' => 0, 'y' => 0, 'width' => 320, 'height' => 200], + ))); + } + + public function testToHaveScreenshotClosesTheTracingGroupOnFailure(): void + { + $path = $this->snapshotDir().'/page.png'; + file_put_contents($path, 'baseline'); + $_SERVER[ScreenshotExpectation::UPDATE_ENV] = 'none'; + + $transport = $this->createStub(TransportInterface::class); + $transport->method('send')->willReturn(['actual' => base64_encode('actual'), 'errorMessage' => 'different']); + + $tracing = $this->createMock(TracingInterface::class); + $tracing->expects($this->once())->method('groupEnd'); + + $page = new Page($transport, $this->createStub(BrowserContextInterface::class), 'page-id', new PlaywrightConfig()); + + $this->expectException(AssertionException::class); + + try { + (new PageAssertions($page, $tracing))->toHaveScreenshot($path); + } finally { + foreach (glob(getcwd().'/test-failures/snapshots/page-*', GLOB_ONLYDIR) ?: [] as $directory) { + foreach (glob($directory.'/*') ?: [] as $file) { + unlink($file); + } + rmdir($directory); + } + @rmdir(getcwd().'/test-failures/snapshots'); + } + } + + private function snapshotDir(): string + { + $dir = sys_get_temp_dir().'/pw-assertions-'.bin2hex(random_bytes(6)); + mkdir($dir); + $this->dirs[] = $dir; + + return $dir; + } + + protected function tearDown(): void + { + unset($_SERVER[ScreenshotExpectation::UPDATE_ENV]); + foreach ($this->dirs as $dir) { + foreach (glob($dir.'/*') ?: [] as $file) { + unlink($file); + } + rmdir($dir); + } + } } diff --git a/tests/Unit/Locator/LocatorTest.php b/tests/Unit/Locator/LocatorTest.php index 7800a32..cb5f224 100644 --- a/tests/Unit/Locator/LocatorTest.php +++ b/tests/Unit/Locator/LocatorTest.php @@ -288,6 +288,15 @@ public function testNormalizeReturnsALocatorOnTheResolvedSelector(): void $this->assertNotSame($this->locator, $normalized); } + public function testNormalizeKeepsThePageOfTheLocator(): void + { + $page = $this->createStub(PageInterface::class); + $locator = new Locator($this->transport, 'page1', '.element', null, null, [], $page); + $this->transport->method('send')->willReturn(['value' => 'internal:role=button[name="Save"i]']); + + $this->assertSame($page, $locator->normalize()->page()); + } + public function testNormalizeRejectsAResponseWithoutASelector(): void { $this->transport @@ -1209,6 +1218,74 @@ public function testDerivedLocatorsCarryThePageAlong(): void $this->assertSame($page, $locator->contentFrame()->locator('.inner')->page()); } + public function testExpectScreenshotSendsTheBaselineAndOptions(): void + { + $this->transport->expects($this->once()) + ->method('send') + ->with([ + 'expected' => base64_encode('baseline'), + 'options' => ['threshold' => 0.1, 'timeout' => 1000], + 'action' => 'locator.expectScreenshot', + 'pageId' => 'page1', + 'selector' => '.element', + ]) + ->willReturn([ + 'actual' => base64_encode('actual'), + 'previous' => null, + 'diff' => base64_encode('diff'), + 'errorMessage' => '12 pixels (ratio 0.01 of all image pixels) are different.', + 'timedOut' => false, + ]); + + $comparison = $this->locator->expectScreenshot('baseline', ['threshold' => 0.1, 'timeout' => 1000]); + + $this->assertSame('actual', $comparison->actual); + $this->assertSame('diff', $comparison->diff); + $this->assertSame('12 pixels (ratio 0.01 of all image pixels) are different.', $comparison->errorMessage); + $this->assertFalse($comparison->matches()); + } + + public function testExpectScreenshotWithoutBaselineAsksForAStableScreenshot(): void + { + $this->transport->expects($this->once()) + ->method('send') + ->with($this->callback(static fn (array $payload): bool => null === $payload['expected'] && [] === $payload['options'])) + ->willReturn(['actual' => base64_encode('stable'), 'errorMessage' => null, 'timedOut' => false]); + + $comparison = $this->locator->expectScreenshot(null); + + $this->assertSame('stable', $comparison->actual); + $this->assertTrue($comparison->matches()); + } + + public function testExpectScreenshotInAFrameSendsTheFrameSelector(): void + { + $locator = new Locator($this->transport, 'page1', '.inner', 'iframe#preview'); + + $this->transport->expects($this->once()) + ->method('send') + ->with($this->callback(static fn (array $payload): bool => 'iframe#preview' === $payload['frameSelector'] && '.inner' === $payload['selector'])) + ->willReturn(['errorMessage' => null]); + + $this->assertTrue($locator->expectScreenshot('baseline')->matches()); + } + + public function testExpectScreenshotSurfacesBridgeErrors(): void + { + $this->transport->method('send')->willReturn(['error' => 'page._expectScreenshot is not a function']); + + $this->expectException(PlaywrightException::class); + $this->expectExceptionMessage('page._expectScreenshot is not a function'); + + $this->locator->expectScreenshot('baseline'); + } + + public function testGetFrameSelector(): void + { + $this->assertNull($this->locator->getFrameSelector()); + $this->assertSame('iframe#preview', (new Locator($this->transport, 'page1', '.inner', 'iframe#preview'))->getFrameSelector()); + } + /** * Rebinds $this->locator to the '.items' selector used by the filter and * combinator tests, which need a different base selector than setUp(). diff --git a/tests/Unit/Page/PageTest.php b/tests/Unit/Page/PageTest.php index 4747e77..fe8f3cb 100644 --- a/tests/Unit/Page/PageTest.php +++ b/tests/Unit/Page/PageTest.php @@ -1235,6 +1235,39 @@ public function testQueriedFramesRetainTheirOriginatingPage(): void $this->assertSame($this->page, $frame->page()); } + public function testExpectScreenshotSendsAPageCommand(): void + { + $this->transport->expects($this->once()) + ->method('send') + ->with([ + 'expected' => base64_encode('baseline'), + 'options' => ['fullPage' => true, 'timeout' => 1000], + 'action' => 'page.expectScreenshot', + 'pageId' => 'page-id', + ]) + ->willReturn(['actual' => null, 'diff' => null, 'errorMessage' => null, 'timedOut' => false]); + + $this->assertTrue($this->page->expectScreenshot('baseline', ['fullPage' => true, 'timeout' => 1000])->matches()); + } + + public function testExpectScreenshotWithoutBaselineReturnsTheStableScreenshot(): void + { + $this->transport->expects($this->once()) + ->method('send') + ->with($this->callback(static fn (array $payload): bool => null === $payload['expected'])) + ->willReturn(['actual' => base64_encode('stable'), 'errorMessage' => null, 'timedOut' => false]); + + $this->assertSame('stable', $this->page->expectScreenshot(null)->actual); + } + + public function testFrameLocatorLocatorsKeepTheirFrameForMasks(): void + { + $locator = $this->page->frameLocator('iframe#preview')->locator('.date'); + + $this->assertInstanceOf(Locator::class, $locator); + $this->assertSame('iframe#preview', $locator->getFrameSelector()); + } + private function createPage(string $pageId = 'page-1'): Page { return new Page($this->transport, $this->context, $pageId, new PlaywrightConfig()); diff --git a/tests/Unit/Screenshot/ScreenshotComparisonTest.php b/tests/Unit/Screenshot/ScreenshotComparisonTest.php new file mode 100644 index 0000000..828b7dc --- /dev/null +++ b/tests/Unit/Screenshot/ScreenshotComparisonTest.php @@ -0,0 +1,58 @@ + base64_encode('actual'), + 'previous' => base64_encode('previous'), + 'diff' => base64_encode('diff'), + 'errorMessage' => '3 pixels are different.', + 'timedOut' => false, + ]); + + $this->assertSame('actual', $comparison->actual); + $this->assertSame('previous', $comparison->previous); + $this->assertSame('diff', $comparison->diff); + $this->assertSame('3 pixels are different.', $comparison->errorMessage); + $this->assertFalse($comparison->timedOut); + $this->assertFalse($comparison->matches()); + } + + public function testMatchesWhenTheBridgeReportsNoError(): void + { + $comparison = ScreenshotComparison::fromResponse(['actual' => null, 'diff' => null, 'errorMessage' => null, 'timedOut' => false]); + + $this->assertTrue($comparison->matches()); + $this->assertNull($comparison->actual); + } + + public function testIgnoresMalformedValues(): void + { + $comparison = ScreenshotComparison::fromResponse(['actual' => '***', 'errorMessage' => '', 'timedOut' => 'yes']); + + $this->assertNull($comparison->actual); + $this->assertNull($comparison->errorMessage); + $this->assertFalse($comparison->timedOut); + } +} diff --git a/tests/Unit/Testing/ExpectFactoryTest.php b/tests/Unit/Testing/ExpectFactoryTest.php index bb0c072..35bdf0d 100644 --- a/tests/Unit/Testing/ExpectFactoryTest.php +++ b/tests/Unit/Testing/ExpectFactoryTest.php @@ -18,14 +18,22 @@ use PHPUnit\Framework\Attributes\CoversFunction; use PHPUnit\Framework\Attributes\CoversTrait; use PHPUnit\Framework\TestCase; +use Playwright\Assertions\Failure\AssertionException; +use Playwright\Assertions\Internal\ScreenshotExpectation; use Playwright\Browser\BrowserContextInterface; +use Playwright\Browser\BrowserInterface; +use Playwright\Browser\BrowserType; +use Playwright\Configuration\PlaywrightConfig; +use Playwright\Locator\Locator; use Playwright\Locator\LocatorInterface; +use Playwright\Page\Page; use Playwright\Page\PageInterface; use Playwright\Testing\Expect; use Playwright\Testing\ExpectDecorator; use Playwright\Testing\ExpectInterface; use Playwright\Testing\PlaywrightTestCaseTrait; use Playwright\Tracing\TracingInterface; +use Playwright\Transport\TransportInterface; use function Playwright\Testing\expect; @@ -127,4 +135,286 @@ public function testNegationDoesNotLeakToTheNextAssertion(): void $expect->not()->toBeVisible(); $expect->toBeVisible(); } + + /** @var list */ + private array $dirs = []; + + public function testRelativeScreenshotNamesGetThePlatformSuffix(): void + { + $dir = $this->snapshotDir(); + + $this->recordBaseline(new Expect($this->pageReturning('stable'), null, $dir), 'card.png'); + + $this->assertSame('stable', file_get_contents($dir.'/card-'.self::platform().'.png')); + } + + public function testScreenshotNamesWithoutExtensionArePng(): void + { + $dir = $this->snapshotDir(); + + $this->recordBaseline(new Expect($this->pageReturning('stable'), null, $dir), 'card'); + + $this->assertFileExists($dir.'/card-'.self::platform().'.png'); + } + + public function testScreenshotNamesCarryTheBrowserOfThePage(): void + { + $dir = $this->snapshotDir(); + + $this->recordBaseline(new Expect($this->pageReturning('stable', browser: BrowserType::FIREFOX), null, $dir), 'card.png'); + + $this->assertFileExists($dir.'/card-firefox-'.self::platform().'.png'); + } + + public function testLocatorScreenshotNamesCarryTheBrowserOfTheirPage(): void + { + $dir = $this->snapshotDir(); + $transport = $this->transportReturning('stable', 'webp', 'locator.expectScreenshot'); + $page = new Page($transport, $this->contextOf(BrowserType::WEBKIT), 'page-id', new PlaywrightConfig()); + $locator = new Locator($transport, 'page-id', '#card', null, null, [], $page); + + $this->recordBaseline(new Expect($locator, null, $dir), 'card.webp'); + + $this->assertFileExists($dir.'/card-webkit-'.self::platform().'.webp'); + } + + public function testWebpScreenshotNamesKeepTheirExtension(): void + { + $dir = $this->snapshotDir(); + + $this->recordBaseline(new Expect($this->pageReturning('stable', type: 'webp'), null, $dir), 'card.webp'); + + $this->assertFileExists($dir.'/card-'.self::platform().'.webp'); + } + + public function testAbsoluteScreenshotPathsAreUsedAsIs(): void + { + $path = $this->snapshotDir().'/exact.png'; + file_put_contents($path, 'baseline'); + + (new Expect($this->pageReturning(null), null, '/elsewhere'))->toHaveScreenshot($path); + + $this->assertSame('baseline', file_get_contents($path)); + } + + public function testWindowsScreenshotPathsAreUsedAsIs(): void + { + $_SERVER[ScreenshotExpectation::UPDATE_ENV] = 'none'; + + $this->expectException(AssertionException::class); + $this->expectExceptionMessage('A snapshot doesn\'t exist at C:\\snapshots\\card.png.'); + + (new Expect($this->pageReturning('stable'), null, '/elsewhere'))->toHaveScreenshot('C:\\snapshots\\card.png'); + } + + public function testUncScreenshotPathsAreUsedAsIs(): void + { + $_SERVER[ScreenshotExpectation::UPDATE_ENV] = 'none'; + + $this->expectException(AssertionException::class); + $this->expectExceptionMessage('A snapshot doesn\'t exist at \\\\server\\share\\card.png.'); + + (new Expect($this->pageReturning('stable'), null, '/elsewhere'))->toHaveScreenshot('\\\\server\\share\\card.png'); + } + + public function testScreenshotsWithoutSnapshotDirectoryResolveInTheWorkingDirectory(): void + { + $dir = $this->snapshotDir(); + $cwd = (string) getcwd(); + chdir($dir); + + try { + $this->recordBaseline(new Expect($this->pageReturning('stable')), 'card.png'); + } finally { + chdir($cwd); + } + + $this->assertFileExists($dir.'/card-'.self::platform().'.png'); + } + + public function testTheDecoratorCountsTheScreenshotAssertion(): void + { + $path = $this->snapshotDir().'/card.png'; + file_put_contents($path, 'baseline'); + + $harness = new class('harness') extends TestCase { + }; + + (new ExpectDecorator(new Expect($this->pageReturning(null)), $harness))->toHaveScreenshot($path); + + $this->assertSame(1, $harness->numberOfAssertionsPerformed()); + } + + public function testFailureImagesGoToADirectoryPerBrowser(): void + { + $dir = $this->snapshotDir(); + $baseline = $dir.'/card-firefox-'.self::platform().'.png'; + file_put_contents($baseline, 'baseline'); + $_SERVER[ScreenshotExpectation::UPDATE_ENV] = 'none'; + + $transport = $this->createStub(TransportInterface::class); + $transport->method('send')->willReturn(['actual' => base64_encode('actual'), 'diff' => base64_encode('diff'), 'errorMessage' => 'different']); + $page = new Page($transport, $this->contextOf(BrowserType::FIREFOX), 'page-id', new PlaywrightConfig()); + + try { + (new Expect($page, null, $dir, $dir.'/CardTest-testCard'))->toHaveScreenshot('card.png'); + $this->fail('A mismatch must fail the assertion.'); + } catch (AssertionException $e) { + $this->assertNotNull($e->actual); + $out = dirname($e->actual); + } + + $this->assertStringStartsWith($dir.'/CardTest-testCard-firefox/', $out); + $this->assertSame(['card-actual.png', 'card-diff.png', 'card-expected.png'], array_map('basename', glob($out.'/*') ?: [])); + foreach (glob($out.'/*') ?: [] as $file) { + unlink($file); + } + rmdir($out); + rmdir(dirname($out)); + } + + public function testALocatorWithoutPageKeepsThePlatformOnly(): void + { + $dir = $this->snapshotDir(); + $transport = $this->transportReturning('stable', 'png', 'locator.expectScreenshot'); + + $this->recordBaseline(new Expect(new Locator($transport, 'page-id', '#card'), null, $dir), 'card.png'); + + $this->assertFileExists($dir.'/card-'.self::platform().'.png'); + } + + public function testTheTraitStoresSnapshotsNextToTheTestFile(): void + { + $harness = new class('harness') extends TestCase { + use PlaywrightTestCaseTrait; + }; + + $this->assertSame(__FILE__.'-snapshots', (new \ReflectionMethod($harness, 'snapshotDirectory'))->invoke($harness)); + } + + public function testTheTraitGivesEachTestItsOwnOutputDirectory(): void + { + $first = new class('testCard') extends TestCase { + use PlaywrightTestCaseTrait; + }; + $second = new class('testButton') extends TestCase { + use PlaywrightTestCaseTrait; + }; + + $method = new \ReflectionMethod($first, 'testOutputDirectory'); + $firstDir = $method->invoke($first); + $secondDir = (new \ReflectionMethod($second, 'testOutputDirectory'))->invoke($second); + + $this->assertStringStartsWith(getcwd().'/test-failures/', $firstDir); + $this->assertStringEndsWith('-testCard', $firstDir); + $this->assertStringEndsWith('-testButton', $secondDir); + $this->assertDoesNotMatchRegularExpression('~[^A-Za-z0-9._-]~', basename($firstDir)); + } + + public function testTheTraitPassesTheSnapshotDirectoryToExpect(): void + { + $dir = $this->snapshotDir(); + $page = $this->pageReturning('stable'); + + $context = $this->createStub(BrowserContextInterface::class); + $context->method('tracing')->willReturn($this->createStub(TracingInterface::class)); + + $harness = new class('harness') extends TestCase { + use PlaywrightTestCaseTrait; + + public string $dir = ''; + + public function callExpect(BrowserContextInterface $context, PageInterface $subject): ExpectInterface + { + $this->context = $context; + + return $this->expect($subject); + } + + protected function snapshotDirectory(): string + { + return $this->dir; + } + }; + $harness->dir = $dir; + + try { + $harness->callExpect($context, $page)->toHaveScreenshot('card.png'); + } catch (AssertionException) { + } + + $this->assertFileExists($dir.'/card-'.self::platform().'.png'); + } + + protected function tearDown(): void + { + unset($_SERVER[ScreenshotExpectation::UPDATE_ENV]); + foreach ($this->dirs as $dir) { + foreach (glob($dir.'/*') ?: [] as $file) { + unlink($file); + } + rmdir($dir); + } + } + + private function recordBaseline(Expect $expect, string $name): void + { + try { + $expect->toHaveScreenshot($name); + $this->fail('Recording a baseline must fail the assertion.'); + } catch (AssertionException $e) { + $this->assertStringContainsString('writing actual', $e->getMessage()); + } + } + + /** + * A page whose bridge returns $actual for every screenshot comparison. + */ + private function pageReturning(?string $actual, string $type = 'png', ?BrowserType $browser = null): Page + { + return new Page($this->transportReturning($actual, $type, 'page.expectScreenshot'), $this->contextOf($browser), 'page-id', new PlaywrightConfig()); + } + + private function transportReturning(?string $actual, string $type, string $action): TransportInterface + { + $transport = $this->createStub(TransportInterface::class); + $transport->method('send')->willReturnCallback(static function (array $payload) use ($actual, $type, $action): array { + if ($action !== $payload['action'] || $type !== $payload['options']['type']) { + throw new \UnexpectedValueException('Unexpected command: '.json_encode($payload)); + } + + return ['actual' => null === $actual ? null : base64_encode($actual), 'errorMessage' => null, 'timedOut' => false]; + }); + + return $transport; + } + + /** + * A context without browser stands for a persistent context. + */ + private function contextOf(?BrowserType $browser): BrowserContextInterface + { + $context = $this->createStub(BrowserContextInterface::class); + if (null !== $browser) { + $instance = $this->createStub(BrowserInterface::class); + $instance->method('browserType')->willReturn($browser); + $context->method('browser')->willReturn($instance); + } + + return $context; + } + + private function snapshotDir(): string + { + $dir = sys_get_temp_dir().'/pw-expect-'.bin2hex(random_bytes(6)); + mkdir($dir); + $this->dirs[] = $dir; + + return $dir; + } + + private static function platform(): string + { + return 'Windows' === \PHP_OS_FAMILY ? 'win32' : strtolower(\PHP_OS_FAMILY); + } }