diff --git a/internal/command/request_param.go b/internal/command/request_param.go index 210eec84..1e8943fe 100644 --- a/internal/command/request_param.go +++ b/internal/command/request_param.go @@ -26,8 +26,8 @@ func createRequestParam(ctx context.Context, cfg *config.Config, files []string, }) } - // Splitting files by example is only supported for rspec, cucumber, and pytest runners - if runner.Name() != "RSpec" && runner.Name() != "Cucumber" && runner.Name() != "pytest" { + // Splitting files by example is only supported for rspec, cucumber, pytest, and playwright runners + if runner.Name() != "RSpec" && runner.Name() != "Cucumber" && runner.Name() != "pytest" && runner.Name() != "Playwright" { params := api.TestPlanParams{ Identifier: cfg.Identifier, Parallelism: cfg.Parallelism, diff --git a/internal/command/run_test.go b/internal/command/run_test.go index 5e177908..5a2c0dc7 100644 --- a/internal/command/run_test.go +++ b/internal/command/run_test.go @@ -753,7 +753,7 @@ func TestCreateRequestParams_NonRSpec(t *testing.T) { defer svr.Close() runners := []TestRunner{ - runner.Jest{}, runner.Playwright{}, runner.Cypress{}, + runner.Jest{}, runner.Cypress{}, } for _, r := range runners { diff --git a/internal/runner/playwright.go b/internal/runner/playwright.go index fd1ce6aa..176d8eab 100644 --- a/internal/runner/playwright.go +++ b/internal/runner/playwright.go @@ -7,6 +7,7 @@ import ( "os" "os/exec" "slices" + "strings" "github.com/buildkite/test-engine-client/internal/debug" "github.com/buildkite/test-engine-client/internal/plan" @@ -162,8 +163,77 @@ func (p Playwright) GetFiles() ([]string, error) { return files, nil } +// GetExamples returns an array of test examples within the given files. func (p Playwright) GetExamples(files []string) ([]plan.TestCase, error) { - return nil, fmt.Errorf("not supported in Playwright") + if len(files) == 0 { + return []plan.TestCase{}, nil + } + + cmdName, cmdArgs, err := p.commandNameAndArgs(p.TestCommand, files) + if err != nil { + return []plan.TestCase{}, err + } + + cmdArgs = append(cmdArgs, "--list", "--reporter=json") + + debug.Printf("Running `%s %s` to list tests", cmdName, strings.Join(cmdArgs, " ")) + + // We use Output() instead of CombinedOutput() because the JSON reporter + // outputs to stdout, and mixing stderr would corrupt the JSON. + output, err := exec.Command(cmdName, cmdArgs...).Output() + if err != nil { + // Include stderr in error message for debugging + if exitErr, ok := err.(*exec.ExitError); ok { + return []plan.TestCase{}, fmt.Errorf("failed to list playwright tests: %s", exitErr.Stderr) + } + return []plan.TestCase{}, fmt.Errorf("failed to list playwright tests: %w", err) + } + + var report PlaywrightReport + if err := json.Unmarshal(output, &report); err != nil { + return []plan.TestCase{}, fmt.Errorf("failed to parse playwright test list output: %s", err) + } + + var testCases []plan.TestCase + for _, suite := range report.Suites { + testCases = append(testCases, p.getTestCasesFromSuite(suite, suite.Title)...) + } + + return testCases, nil +} + +// getTestCasesFromSuite recursively traverses the Playwright report suite and returns all test cases. +// Playwright's report format is a tree structure, where each suite can contain multiple specs and sub-suites. +func (p Playwright) getTestCasesFromSuite(suite PlaywrightReportSuite, suiteName string) []plan.TestCase { + var testCases []plan.TestCase + + for _, spec := range suite.Specs { + testCases = append(testCases, mapSpecToTestCase(spec, suiteName)) + } + + for _, subSuite := range suite.Suites { + testCases = append(testCases, p.getTestCasesFromSuite(subSuite, fmt.Sprintf("%s %s", suiteName, subSuite.Title))...) + } + + return testCases +} + +// mapSpecToTestCase converts a Playwright spec to a plan.TestCase. +// The scope has to match with the scope generated by Buildkite test collector. +// In Buildkite test collector, the scope is generated using Playwright built-in reporter function, titlePath(). +// titlePath function returns an array of suite's title from the root suite down to the current test, +// which is then joined with a space separator to form the scope. +// For more details, see: +// [Buildkite Test Collector - Playwright implementation](https://github.com/buildkite/test-collector-javascript/blob/42b803a618a15a07edf0169038ef4b5eba88f98d/playwright/reporter.js#L47) +// [Playwright titlePath implementation](https://github.com/microsoft/playwright/blob/523e50088a7f982dd96aacdb260dfbd1189159b1/packages/playwright/src/common/test.ts#L126) +// [Playwright suite structure](https://playwright.dev/docs/api/class-suite) +func mapSpecToTestCase(spec PlaywrightSpec, suiteName string) plan.TestCase { + projectName := spec.Tests[0].ProjectName + return plan.TestCase{ + Name: spec.Title, + Path: fmt.Sprintf("%s:%d", spec.File, spec.Line), + Scope: fmt.Sprintf(" %s %s %s", projectName, suiteName, spec.Title), + } } type PlaywrightTest struct { diff --git a/internal/runner/playwright_test.go b/internal/runner/playwright_test.go index 5b49e9c1..2e979198 100644 --- a/internal/runner/playwright_test.go +++ b/internal/runner/playwright_test.go @@ -266,3 +266,69 @@ func TestPlaywrightGetFiles(t *testing.T) { t.Errorf("Playwright.GetFiles() diff (-got +want):\n%s", diff) } } + +func TestPlaywrightGetExamples(t *testing.T) { + changeCwd(t, "./testdata/playwright") + + playwright := NewPlaywright(RunnerConfig{ + // Use npx directly because yarn adds prefix output to stdout that corrupts JSON parsing + TestCommand: "npx playwright test", + }) + + files := []string{"tests/example.spec.js"} + got, err := playwright.GetExamples(files) + if err != nil { + t.Fatalf("Playwright.GetExamples(%v) error = %v", files, err) + } + + // example.spec.js has 2 tests in a "home page" describe block. + // With 2 projects (chromium, firefox), we expect 4 test cases. + want := []plan.TestCase{ + { + Name: "has title", + Path: "example.spec.js:4", + Scope: " chromium example.spec.js home page has title", + }, + { + Name: "has title", + Path: "example.spec.js:4", + Scope: " firefox example.spec.js home page has title", + }, + { + Name: "says hello", + Path: "example.spec.js:10", + Scope: " chromium example.spec.js home page says hello", + }, + { + Name: "says hello", + Path: "example.spec.js:10", + Scope: " firefox example.spec.js home page says hello", + }, + } + + // Sort both slices by scope for consistent comparison + sorter := cmp.Transformer("Sort", func(in []plan.TestCase) []plan.TestCase { + out := append([]plan.TestCase(nil), in...) + slices.SortFunc(out, func(a, b plan.TestCase) int { + return strings.Compare(a.Scope, b.Scope) + }) + return out + }) + + if diff := cmp.Diff(got, want, sorter); diff != "" { + t.Errorf("Playwright.GetExamples(%v) diff (-got +want):\n%s", files, diff) + } +} + +func TestPlaywrightGetExamples_EmptyFiles(t *testing.T) { + playwright := NewPlaywright(RunnerConfig{}) + + got, err := playwright.GetExamples([]string{}) + if err != nil { + t.Errorf("Playwright.GetExamples([]) error = %v", err) + } + + if len(got) != 0 { + t.Errorf("Playwright.GetExamples([]) = %v, want empty slice", got) + } +}