diff --git a/nodejs/docs/api/class-testconfig.mdx b/nodejs/docs/api/class-testconfig.mdx index 9962fa43c00..84425aca51a 100644 --- a/nodejs/docs/api/class-testconfig.mdx +++ b/nodejs/docs/api/class-testconfig.mdx @@ -405,7 +405,7 @@ export default defineConfig({ Added in: v1.10testConfig.metadata -Metadata contains key-value pairs to be included in the report. For example, HTML report will display it as key-value pairs, and JSON report will include metadata serialized as json. +Metadata contains key-value pairs to be included in the report. For example, the JSON report will include metadata serialized as JSON. **Usage** @@ -923,7 +923,7 @@ export default defineConfig({ Whether to update expected snapshots with the actual results produced by the test run. Defaults to `'missing'`. * `'all'` - All tests that are executed will update snapshots. -* `'changed'` - All tests that are executed will update snapshots that did not match. Matching snapshots will not be updated. +* `'changed'` - All tests that are executed will update snapshots that did not match. Matching snapshots will not be updated. Also creates missing snapshots. * `'missing'` - Missing snapshots are created, for example when authoring a new test and running it for the first time. This is the default. * `'none'` - No snapshots are updated. diff --git a/nodejs/docs/test-agents.mdx b/nodejs/docs/test-agents.mdx new file mode 100644 index 00000000000..15619bb6f0b --- /dev/null +++ b/nodejs/docs/test-agents.mdx @@ -0,0 +1,318 @@ +--- +id: test-agents +title: "Agents" +--- +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import HTMLCard from '@site/src/components/HTMLCard'; +import ProgressiveImage from '@theme/ProgressiveImage'; + +# Playwright Agents + +## Test Coverage in 1-2-3 + +Playwright’s agentic workflow makes it possible to generate test coverage in three straightforward steps. These steps can be performed independently, manually, or as chained calls in an agentic loop. +1. **Plan**: A planning agent explores the app and produces a test plan in `specs/*.md`. + +2. **Generate**: A generating agent transforms the plan into `tests/*.spec.ts` files. It executes actions live to verify selectors and flows, then emits testing code and assertions. + +3. **Heal**: A healing agent executes the test suite and automatically repairs failing tests by applying diffs in place. + +## 1. Plan + +**Input** +* A clear request (e.g., “Generate a plan for guest checkout.”) +* A live entry point (URL) or a seed Playwright test that sets up the environment +* PRD (optional) + +**Prompt** + +```markdown +Ask `playwright-test-planner` agent to generate a test plan for "Guest Checkout" scenario. +Use `seed.spec.ts` as a seed test for the plan. +``` + +**Output** +* A Markdown test plan saved to `specs/guest-checkout.md`. The plan is human-readable but precise enough for test generation. + +
+ +Example: specs/guest-checkout.md + +```markdown +# Feature: Guest Checkout + +## Purpose +Allow a user to purchase without creating an account. + +## Preconditions +- Test seed `tests/seed.spec.ts`. +- Payment sandbox credentials available via env vars. + +## Scenarios + +### SC-1: Add single item to cart and purchase +**Steps** +1. Open home page. +2. Search for "Wireless Mouse". +3. Open product page and add to cart. +4. Proceed to checkout as guest. +5. Fill shipping and payment details. +6. Confirm order. + +**Expected** +- Cart count increments after item is added. +- Checkout page shows item, price, tax, and total. +- Order confirmation number appears; status is "Processing". + +### SC-2: Tax and shipping recalculation on address change +**Steps** +1. Start checkout with a CA address. +2. Change state to NY. + +**Expected** +- Tax and shipping values recalculate. + +## Data +- Product SKU: `WM-123` +- Payment: sandbox card `4111 1111 1111 1111`, valid expiry, CVV `123`. + +## Methodology +*Optional notes about testing methodology* +``` + +
+ +## 2. Generate + +The generating agent uses the Markdown plan to produce executable Playwright tests. It verifies selectors and assertions live against the application. Playwright supports generation hints and provides a catalog of assertions for efficient structural and behavioral validation. + +**Input** +* Markdown plan from `specs/` + +**Prompt** + +```markdown +Ask `playwright-test-generator` to generate tests for the guest checkout plan under `specs/`. +``` + +**Output** +* A test suite under `tests/` +* Tests may include initial errors that can be healed automatically + +
+ +Example: tests/guest-checkout.spec.ts + +```ts +import { test, expect } from '@playwright/test'; + +test.describe('Guest Checkout', () => { + test('SC-1: add item and purchase', async ({ page }) => { + await page.goto('/'); + await page.getByRole('searchbox', { name: /search/i }).fill('Wireless Mouse'); + await page.getByRole('button', { name: /search/i }).click(); + + await page.getByRole('link', { name: /wireless mouse/i }).click(); + await page.getByRole('button', { name: /add to cart/i }).click(); + + // Assertion: cart badge increments + await expect(page.getByTestId('cart-badge')).toHaveText('1'); + + await page.getByRole('link', { name: /checkout/i }).click(); + await page.getByRole('button', { name: /continue as guest/i }).click(); + + // Fill checkout form + await page.getByLabel('Email').fill(process.env.CHECKOUT_EMAIL!); + await page.getByLabel('Full name').fill('Alex Guest'); + await page.getByLabel('Address').fill('1 Market St'); + await page.getByLabel('City').fill('San Francisco'); + await page.getByLabel('State').selectOption('CA'); + await page.getByLabel('ZIP').fill('94105'); + + // Payment (sandbox) + const frame = page.frameLocator('[data-testid="card-iframe"]'); + await frame.getByLabel('Card number').fill('4111111111111111'); + await frame.getByLabel('MM / YY').fill('12/30'); + await frame.getByLabel('CVC').fill('123'); + + await page.getByRole('button', { name: /pay/i }).click(); + + // Assertions: confirmation invariants + await expect(page).toHaveURL(/\/orders\/\w+\/confirmation/); + await expect(page.getByRole('heading', { name: /thank you/i })).toBeVisible(); + await expect(page.getByTestId('order-status')).toHaveText(/processing/i); + + // Optional visual check + await expect(page.locator('[data-testid="order-summary"]')).toHaveScreenshot(); + }); +}); +``` + +
+ +## 3. Heal + +When a test fails, the healing agent: +* Replays the failing steps +* Inspects the current UI to locate equivalent elements or flows +* Suggests a patch (e.g., locator update, wait adjustment, data fix) +* Re-runs the test until it passes or until guardrails stop the loop + +**Input** +* Failing test name + +**Prompt** + +```markdown +Ask `playwright-test-healer` to fix all failing tests for the guest checkout scenario. +``` + +**Output** +* A passing test, or a skipped test if the functionality is broken + +## Artifacts and Conventions + +Follow a simple, auditable structure: + +```bash +repo/ + .{claude|copilot|vscode|...}/ # agent definitions, tools, guardrails + specs/ # human-readable test plans + checkout-guest.md + account-settings.md + tests/ # generated Playwright tests + seed.spec.ts + checkout-guest.spec.ts + account-settings.spec.ts + playwright.config.ts +``` + +### Agents definitions + +Agent definitions are collections of instructions and MCP tools. They are provided by Playwright and should be regenerated whenever Playwright is updated. + +Example for Claude Code subagents: + +```bash +npx playwright init-agents --claude +``` + +### Specs in `specs/` + +Specs are structured plans describing scenarios in human-readable terms. They include steps, expected outcomes, and data. Specs can start from scratch or extend a seed test. + +### Tests in `tests/` + +Generated Playwright tests, aligned one-to-one with specs wherever feasible. + +### Seed tests `seed.spec.ts` + +Seed tests provide a ready-to-use `page` context to bootstrap execution. + + +[Accessibility]: /api/class-accessibility.mdx "Accessibility" +[Android]: /api/class-android.mdx "Android" +[AndroidDevice]: /api/class-androiddevice.mdx "AndroidDevice" +[AndroidInput]: /api/class-androidinput.mdx "AndroidInput" +[AndroidSocket]: /api/class-androidsocket.mdx "AndroidSocket" +[AndroidWebView]: /api/class-androidwebview.mdx "AndroidWebView" +[APIRequest]: /api/class-apirequest.mdx "APIRequest" +[APIRequestContext]: /api/class-apirequestcontext.mdx "APIRequestContext" +[APIResponse]: /api/class-apiresponse.mdx "APIResponse" +[APIResponseAssertions]: /api/class-apiresponseassertions.mdx "APIResponseAssertions" +[Browser]: /api/class-browser.mdx "Browser" +[BrowserContext]: /api/class-browsercontext.mdx "BrowserContext" +[BrowserServer]: /api/class-browserserver.mdx "BrowserServer" +[BrowserType]: /api/class-browsertype.mdx "BrowserType" +[CDPSession]: /api/class-cdpsession.mdx "CDPSession" +[Clock]: /api/class-clock.mdx "Clock" +[ConsoleMessage]: /api/class-consolemessage.mdx "ConsoleMessage" +[Coverage]: /api/class-coverage.mdx "Coverage" +[Dialog]: /api/class-dialog.mdx "Dialog" +[Download]: /api/class-download.mdx "Download" +[Electron]: /api/class-electron.mdx "Electron" +[ElectronApplication]: /api/class-electronapplication.mdx "ElectronApplication" +[ElementHandle]: /api/class-elementhandle.mdx "ElementHandle" +[FileChooser]: /api/class-filechooser.mdx "FileChooser" +[Frame]: /api/class-frame.mdx "Frame" +[FrameLocator]: /api/class-framelocator.mdx "FrameLocator" +[GenericAssertions]: /api/class-genericassertions.mdx "GenericAssertions" +[JSHandle]: /api/class-jshandle.mdx "JSHandle" +[Keyboard]: /api/class-keyboard.mdx "Keyboard" +[Locator]: /api/class-locator.mdx "Locator" +[LocatorAssertions]: /api/class-locatorassertions.mdx "LocatorAssertions" +[Logger]: /api/class-logger.mdx "Logger" +[Mouse]: /api/class-mouse.mdx "Mouse" +[Page]: /api/class-page.mdx "Page" +[PageAssertions]: /api/class-pageassertions.mdx "PageAssertions" +[Playwright]: /api/class-playwright.mdx "Playwright" +[PlaywrightAssertions]: /api/class-playwrightassertions.mdx "PlaywrightAssertions" +[Request]: /api/class-request.mdx "Request" +[Response]: /api/class-response.mdx "Response" +[Route]: /api/class-route.mdx "Route" +[Selectors]: /api/class-selectors.mdx "Selectors" +[SnapshotAssertions]: /api/class-snapshotassertions.mdx "SnapshotAssertions" +[TimeoutError]: /api/class-timeouterror.mdx "TimeoutError" +[Touchscreen]: /api/class-touchscreen.mdx "Touchscreen" +[Tracing]: /api/class-tracing.mdx "Tracing" +[Video]: /api/class-video.mdx "Video" +[WebError]: /api/class-weberror.mdx "WebError" +[WebSocket]: /api/class-websocket.mdx "WebSocket" +[WebSocketRoute]: /api/class-websocketroute.mdx "WebSocketRoute" +[Worker]: /api/class-worker.mdx "Worker" +[Fixtures]: /api/class-fixtures.mdx "Fixtures" +[FullConfig]: /api/class-fullconfig.mdx "FullConfig" +[FullProject]: /api/class-fullproject.mdx "FullProject" +[Location]: /api/class-location.mdx "Location" +[Test]: /api/class-test.mdx "Test" +[TestConfig]: /api/class-testconfig.mdx "TestConfig" +[TestInfo]: /api/class-testinfo.mdx "TestInfo" +[TestInfoError]: /api/class-testinfoerror.mdx "TestInfoError" +[TestOptions]: /api/class-testoptions.mdx "TestOptions" +[TestProject]: /api/class-testproject.mdx "TestProject" +[TestStepInfo]: /api/class-teststepinfo.mdx "TestStepInfo" +[WorkerInfo]: /api/class-workerinfo.mdx "WorkerInfo" +[Reporter]: /api/class-reporter.mdx "Reporter" +[Suite]: /api/class-suite.mdx "Suite" +[TestCase]: /api/class-testcase.mdx "TestCase" +[TestError]: /api/class-testerror.mdx "TestError" +[TestResult]: /api/class-testresult.mdx "TestResult" +[TestStep]: /api/class-teststep.mdx "TestStep" +[Element]: https://developer.mozilla.org/en-US/docs/Web/API/element "Element" +[EvaluationArgument]: /evaluating.mdx#evaluation-argument "EvaluationArgument" +[Promise]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise "Promise" +[iterator]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Iteration_protocols "Iterator" +[origin]: https://developer.mozilla.org/en-US/docs/Glossary/Origin "Origin" +[selector]: https://developer.mozilla.org/en-US/docs/Web/CSS/CSS_Selectors "selector" +[Serializable]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/JSON/stringify#Description "Serializable" +[UIEvent.detail]: https://developer.mozilla.org/en-US/docs/Web/API/UIEvent/detail "UIEvent.detail" +[UnixTime]: https://en.wikipedia.org/wiki/Unix_time "Unix Time" +[xpath]: https://developer.mozilla.org/en-US/docs/Web/XPath "xpath" + +[Array]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array "Array" +[boolean]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Data_structures#Boolean_type "Boolean" +[Buffer]: https://nodejs.org/api/buffer.html#buffer_class_buffer "Buffer" +[ChildProcess]: https://nodejs.org/api/child_process.html "ChildProcess" +[Date]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Date "Date" +[Error]: https://nodejs.org/api/errors.html#errors_class_error "Error" +[EventEmitter]: https://nodejs.org/api/events.html#events_class_eventemitter "EventEmitter" +[function]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Function "Function" +[FormData]: https://developer.mozilla.org/en-US/docs/Web/API/FormData "FormData" +[Map]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Map "Map" +[Metadata]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Object "Object<string, any>" +[null]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/null "null" +[number]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Data_structures#Number_type "Number" +[Object]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Object "Object" +[Promise]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise "Promise" +[Readable]: https://nodejs.org/api/stream.html#stream_class_stream_readable "Readable" +[ReadStream]: https://nodejs.org/api/fs.html#class-fsreadstream "ReadStream" +[RegExp]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/RegExp "RegExp" +[string]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Data_structures#String_type "string" +[void]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/undefined "void" +[URL]: https://nodejs.org/api/url.html "URL" +[URLSearchParams]: https://developer.mozilla.org/en-US/docs/Web/API/URLSearchParams "URLSearchParams" + +[all available image tags]: https://mcr.microsoft.com/en-us/product/playwright/about "all available image tags" +[Microsoft Artifact Registry]: https://mcr.microsoft.com/en-us/product/playwright/about "Microsoft Artifact Registry" +[Dockerfile.noble]: https://github.com/microsoft/playwright/blob/main/utils/docker/Dockerfile.noble "Dockerfile.noble" diff --git a/nodejs/docs/test-configuration.mdx b/nodejs/docs/test-configuration.mdx index 6d7a5abbcf8..b0d71c4a346 100644 --- a/nodejs/docs/test-configuration.mdx +++ b/nodejs/docs/test-configuration.mdx @@ -1,6 +1,6 @@ --- id: test-configuration -title: "Test configuration" +title: "Configuration" --- import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; diff --git a/nodejs/docs/test-reporters.mdx b/nodejs/docs/test-reporters.mdx index 51d5174dfa0..ac8fcab1c24 100644 --- a/nodejs/docs/test-reporters.mdx +++ b/nodejs/docs/test-reporters.mdx @@ -253,6 +253,7 @@ HTML report supports the following configuration options and environment variabl | `PLAYWRIGHT_HTML_HOST` | `host` | When report opens in the browser, it will be served bound to this hostname. | `localhost` | `PLAYWRIGHT_HTML_PORT` | `port` | When report opens in the browser, it will be served on this port. | `9323` or any available port when `9323` is not available. | `PLAYWRIGHT_HTML_ATTACHMENTS_BASE_URL` | `attachmentsBaseURL` | A separate location where attachments from the `data` subdirectory are uploaded. Only needed when you upload report and `data` separately to different locations. | `data/` +| `PLAYWRIGHT_HTML_NO_COPY_PROMPT` | `noCopyPrompt` | If true, disable rendering of the Copy prompt for errors. Supports `true`, `1`, `false`, and `0`. | `false` | `PLAYWRIGHT_HTML_NO_SNIPPETS` | `noSnippets` | If true, disable rendering code snippets in the action log. If there is a top level error, that report section with code snippet will still render. Supports `true`, `1`, `false`, and `0`. | `false` ### Blob reporter diff --git a/nodejs/docs/test-use-options.mdx b/nodejs/docs/test-use-options.mdx index 4840d67a296..c304094607f 100644 --- a/nodejs/docs/test-use-options.mdx +++ b/nodejs/docs/test-use-options.mdx @@ -1,6 +1,6 @@ --- id: test-use-options -title: "Test use options" +title: "Configuration (use)" --- import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; diff --git a/nodejs/sidebars.js b/nodejs/sidebars.js index bce1506ac2c..b0b7f6a797f 100644 --- a/nodejs/sidebars.js +++ b/nodejs/sidebars.js @@ -20,10 +20,11 @@ module.exports = { type: 'category', label: 'Playwright Test', items: [ - { type: 'doc', id: 'test-configuration' }, - { type: 'doc', id: 'test-use-options' }, + { type: 'doc', id: 'test-agents' }, { type: 'doc', id: 'test-annotations' }, { type: 'doc', id: 'test-cli' }, + { type: 'doc', id: 'test-configuration' }, + { type: 'doc', id: 'test-use-options' }, { type: 'doc', id: 'emulation' }, { type: 'doc', id: 'test-fixtures' }, { type: 'doc', id: 'test-global-setup-teardown' }, diff --git a/src/generator.js b/src/generator.js index 5f3932fe5b8..d2b37ba10bd 100644 --- a/src/generator.js +++ b/src/generator.js @@ -29,7 +29,7 @@ const { generateTabGroups, renderHTMLCard } = require('./format_utils'); /** @typedef {import('./markdown').MarkdownNode} MarkdownNode */ /** @export @typedef {{ name: string, link: string, usages: string[], args: docs.Member[], signatures?: string[] }} FormattedMember */ -const commonSnippets = new Set(['txt', 'html', 'xml', 'yml', 'yaml', 'json', 'groovy', 'html', 'bash', 'sh', 'ini', 'Dockerfile', 'css']); +const commonSnippets = new Set(['txt', 'html', 'xml', 'yml', 'yaml', 'json', 'groovy', 'html', 'bash', 'sh', 'ini', 'Dockerfile', 'css', 'markdown']); /** @typedef {"header"|"link"|"usage"} FormatMode */