From bc2f09bf1630f2a61bb64a801e9186107af06d28 Mon Sep 17 00:00:00 2001 From: Tom Meier Date: Wed, 28 May 2025 10:26:13 +1000 Subject: [PATCH 1/4] Initial support for Cucumber test engine client --- .buildkite/Dockerfile | 3 +- README.md | 15 +- bin/setup | 4 + docs/cucumber.md | 58 ++ internal/runner/cucumber.go | 254 ++++++++ internal/runner/cucumber_result_parser.go | 146 +++++ internal/runner/cucumber_test.go | 555 ++++++++++++++++++ internal/runner/detector.go | 4 +- internal/runner/doc.go | 1 + internal/runner/testdata/cucumber/Gemfile | 6 + .../runner/testdata/cucumber/Gemfile.lock | 47 ++ .../cucumber/features/another_feature.feature | 5 + .../cucumber/features/failure.feature | 4 + .../features/simple_scenarios.feature | 25 + .../features/spells/expelliarmus.feature | 4 + .../features/step_definitions/steps.rb | 51 ++ main.go | 4 +- 17 files changed, 1175 insertions(+), 11 deletions(-) create mode 100644 docs/cucumber.md create mode 100644 internal/runner/cucumber.go create mode 100644 internal/runner/cucumber_result_parser.go create mode 100644 internal/runner/cucumber_test.go create mode 100644 internal/runner/testdata/cucumber/Gemfile create mode 100644 internal/runner/testdata/cucumber/Gemfile.lock create mode 100644 internal/runner/testdata/cucumber/features/another_feature.feature create mode 100644 internal/runner/testdata/cucumber/features/failure.feature create mode 100644 internal/runner/testdata/cucumber/features/simple_scenarios.feature create mode 100644 internal/runner/testdata/cucumber/features/spells/expelliarmus.feature create mode 100644 internal/runner/testdata/cucumber/features/step_definitions/steps.rb diff --git a/.buildkite/Dockerfile b/.buildkite/Dockerfile index bc460371..c6dadfbe 100644 --- a/.buildkite/Dockerfile +++ b/.buildkite/Dockerfile @@ -8,7 +8,8 @@ COPY --from=ruby / / COPY --from=cypress / / COPY --from=python / / -RUN gem install rspec +RUN gem install rspec cucumber base64 +RUN gem install bigdecimal -v 3.2.0 RUN yarn global add jest RUN pip install pytest RUN pip install buildkite-test-collector==0.2.0 diff --git a/README.md b/README.md index 0bb98410..eab91960 100644 --- a/README.md +++ b/README.md @@ -4,13 +4,13 @@ Buildkite Test Engine Client (bktec) is an open source tool to orchestrate your bktec supports multiple test runners and offers various features to enhance your testing workflow. Below is a comparison of the features supported by each test runner: -| Feature | RSpec | Jest | Playwright | Cypress | pytest | pants (pytest) | Go test | -| -------------------------------------------------- | :---: | :--: | :---------: | :-----: | :-----: | :------------: | :-----: | -| Filter test files | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | -| Automatically retry failed test | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | -| Split slow files by individual test example | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | -| Mute tests (ignore test failures) | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | -| Skip tests | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | +| Feature | RSpec | Jest | Playwright | Cypress | pytest | pants (pytest) | Go test | Cucumber | +| -------------------------------------------------- | :---: | :--: | :--------: | :-----: | :----: | :------------: | :-----: | :------: | +| Filter test files | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | +| Automatically retry failed test | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | +| Split slow files by individual test example | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | +| Mute tests (ignore test failures) | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | +| Skip tests | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ## Installation The latest version of bktec can be downloaded from https://github.com/buildkite/test-engine-client/releases @@ -63,6 +63,7 @@ To configure the test runner for bktec, please refer to the detailed guides prov - [pytest pants](./docs/pytest-pants.md) - [go test](./docs/gotest.md) - [RSpec](./docs/rspec.md) +- [Cucumber](./docs/cucumber.md) ### Running bktec diff --git a/bin/setup b/bin/setup index cb402864..00e1cb33 100755 --- a/bin/setup +++ b/bin/setup @@ -47,6 +47,10 @@ npx cypress verify cd ../rspec bundle install +# Install Cucumber dependencies +cd ../cucumber +bundle install + # Install various python things, dependencies for pytest test cases if [ -n "$VIRTUAL_ENV" ]; then echo "Python virtual environment is active: $VIRTUAL_ENV" diff --git a/docs/cucumber.md b/docs/cucumber.md new file mode 100644 index 00000000..0805eb8a --- /dev/null +++ b/docs/cucumber.md @@ -0,0 +1,58 @@ +# Using bktec with Cucumber + +To integrate bktec with Cucumber, set the `BUILDKITE_TEST_ENGINE_TEST_RUNNER` environment variable to `cucumber`. Then, specify the `BUILDKITE_TEST_ENGINE_RESULT_PATH` to define where the JSON result should be stored. bktec will instruct Cucumber to output the JSON result to this path, which is necessary for bktec to read the test results for retries and verification purposes. + +```sh +export BUILDKITE_TEST_ENGINE_TEST_RUNNER=cucumber +export BUILDKITE_TEST_ENGINE_RESULT_PATH=tmp/cucumber-result.json +``` + +## Configure test command +By default, bktec runs Cucumber with the following command: + +```sh +bundle exec cucumber --format pretty --format json --out {{resultPath}} {{testExamples}} +``` + +In this command: +- `{{testExamples}}` is replaced by bktec with the list of feature files or scenarios to run. +- `{{resultPath}}` is replaced with the value set in `BUILDKITE_TEST_ENGINE_RESULT_PATH`. + +You can customise this command using the `BUILDKITE_TEST_ENGINE_TEST_CMD` environment variable. + +```sh +export BUILDKITE_TEST_ENGINE_TEST_CMD="bundle exec cucumber --format json --out {{resultPath}} {{testExamples}}" +``` + +> **IMPORTANT** – Make sure your custom command includes `--format json --out {{resultPath}}` so that bktec can parse the results. + +## Filter feature files +By default, bktec runs feature files that match the `features/**/*.feature` pattern. You can customise this pattern using the `BUILDKITE_TEST_ENGINE_TEST_FILE_PATTERN` environment variable. + +```sh +export BUILDKITE_TEST_ENGINE_TEST_FILE_PATTERN=features/login/**/*.feature +``` + +You can also exclude certain directories or files with `BUILDKITE_TEST_ENGINE_TEST_FILE_EXCLUDE_PATTERN`: + +```sh +export BUILDKITE_TEST_ENGINE_TEST_FILE_EXCLUDE_PATTERN=features/experimental +``` + +> **TIP** – The patterns use the same glob syntax as the [zzglob](https://github.com/DrJosh9000/zzglob#pattern-syntax) library. + +## Automatically retry failed scenarios +Use `BUILDKITE_TEST_ENGINE_RETRY_COUNT` to automatically retry failed scenarios. When this variable is set and greater than `0`, failed scenarios will be re-run using the command from `BUILDKITE_TEST_ENGINE_RETRY_CMD` (or the main test command if not set). + +```sh +export BUILDKITE_TEST_ENGINE_RETRY_COUNT=2 +``` + +A typical retry command might look like: + +```sh +export BUILDKITE_TEST_ENGINE_RETRY_CMD="bundle exec cucumber {{testExamples}} --format json --out {{resultPath}}" +``` + +## Split by example +Splitting slow files by individual scenario is supported for Cucumber. When bktec identifies slow files, it can request a plan that splits these files into individual scenarios. The `{{testExamples}}` placeholder in your test command will then be populated with specific `file:line` identifiers for each scenario to be run. diff --git a/internal/runner/cucumber.go b/internal/runner/cucumber.go new file mode 100644 index 00000000..db14fe6c --- /dev/null +++ b/internal/runner/cucumber.go @@ -0,0 +1,254 @@ +package runner + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "os/exec" + "slices" + "strings" + + "github.com/buildkite/test-engine-client/internal/debug" + "github.com/buildkite/test-engine-client/internal/plan" + "github.com/kballard/go-shellquote" +) + +// Cucumber implements TestRunner for Cucumber (Ruby implementation). +// It follows very similar behaviour to the RSpec runner. We rely on the JSON formatter +// so users MUST include `--format json --out {{resultPath}}` in their custom commands. +// +// We treat every Scenario as an individual test case. A scenario is considered failed +// if any step in it failed or has undefined status. "pending" and "skipped" are +// mapped to TestStatusSkipped. + +type Cucumber struct { + RunnerConfig +} + +func NewCucumber(c RunnerConfig) Cucumber { + if c.TestCommand == "" { + // The pretty formatter gives a nice progress bar in the console, the JSON formatter is required for bktec. + c.TestCommand = "cucumber --format pretty --format json --out {{resultPath}} {{testExamples}}" + } + + if c.TestFilePattern == "" { + c.TestFilePattern = "features/**/*.feature" + } + + if c.RetryTestCommand == "" { + c.RetryTestCommand = c.TestCommand + } + + return Cucumber{ + RunnerConfig: c, + } +} + +func (c Cucumber) Name() string { + return "Cucumber" +} + +// GetFiles returns the list of feature files based on include / exclude pattern. +func (c Cucumber) GetFiles() ([]string, error) { + debug.Println("Discovering test files with include pattern:", c.TestFilePattern, "exclude pattern:", c.TestFileExcludePattern) + files, err := discoverTestFiles(c.TestFilePattern, c.TestFileExcludePattern) + debug.Println("Discovered", len(files), "files") + + if err != nil { + return nil, err + } + + if len(files) == 0 { + return nil, fmt.Errorf("no files found with pattern %q and exclude pattern %q", c.TestFilePattern, c.TestFileExcludePattern) + } + + return files, nil +} + +// Run executes the Cucumber command and records results. +func (c Cucumber) Run(result *RunResult, testCases []plan.TestCase, retry bool) error { + command := c.TestCommand + if retry { + command = c.RetryTestCommand + } + + testPaths := make([]string, len(testCases)) + for i, tc := range testCases { + testPaths[i] = tc.Path + } + + commandName, commandArgs, err := c.commandNameAndArgs(command, testPaths) + if err != nil { + return fmt.Errorf("failed to build command: %w", err) + } + + cmd := exec.Command(commandName, commandArgs...) + + err = runAndForwardSignal(cmd) + if ProcessSignaledError := new(ProcessSignaledError); errors.As(err, &ProcessSignaledError) { + return err + } + + report, parseErr := c.ParseReport(c.ResultPath) + if parseErr != nil { + fmt.Println("Buildkite Test Engine Client: Failed to read Cucumber JSON output, tests will not be retried.") + return err + } + + // Iterate scenarios. + for _, feature := range report { + for _, scenario := range feature.Elements { + if scenario.Type != "scenario" { + continue + } + status := scenario.AggregatedStatus() + var testStatus TestStatus + switch status { + case "failed", "undefined", "errored": + testStatus = TestStatusFailed + case "passed": + testStatus = TestStatusPassed + case "pending", "skipped" /* cucumber-js uses skipped */ : + testStatus = TestStatusSkipped + default: + testStatus = TestStatusSkipped + } + + fileLinePath := fmt.Sprintf("%s:%d", feature.URI, scenario.Line) + testCaseForResult := plan.TestCase{ + Identifier: fileLinePath, // Use file:line as the primary identifier + Name: scenario.Name, + Scope: feature.Name, + Path: fileLinePath, + } + + result.RecordTestResult(testCaseForResult, testStatus) + } + } + + // Determine if there were any errors outside of scenarios. Cucumber does not + // provide such count – we rely on process exit status already handled above. + + return nil +} + +// CucumberFeature and CucumberElement structs would be defined, likely in a separate parser file. +// For brevity, they are assumed here. + +// mapScenarioToTestCase maps a Cucumber scenario (element) to a plan.TestCase +func mapScenarioToTestCase(featureURI string, scenario CucumberElement) plan.TestCase { + // Cucumber scenarios are identified by file_path:line_number + identifier := fmt.Sprintf("%s:%d", featureURI, scenario.Line) + return plan.TestCase{ + Path: identifier, + Name: scenario.Name, + Identifier: identifier, // Or scenario.ID if it's more suitable and consistently available + } +} + +// GetExamples returns an array of test scenarios within the given feature files. +func (c Cucumber) GetExamples(files []string) ([]plan.TestCase, error) { + if len(files) == 0 { + return []plan.TestCase{}, nil + } + + // Create a temporary file to store the JSON output of the cucumber dry run. + f, err := os.CreateTemp("", "cucumber-dry-run-*.json") + if err != nil { + return nil, fmt.Errorf("failed to create temporary file for cucumber dry run: %w", err) + } + debug.Printf("Created temp file for cucumber dry run: %s", f.Name()) + + defer func() { + closeErr := f.Close() + if closeErr != nil { + debug.Printf("Error closing temp file %s: %v", f.Name(), closeErr) + } + removeErr := os.Remove(f.Name()) + if removeErr != nil { + debug.Printf("Error removing temp file %s: %v", f.Name(), removeErr) + } + }() + + cmdName, _, err := c.commandNameAndArgs(c.TestCommand, files) + if err != nil { + return nil, err + } + + dryRunArgs := append( + []string{"--dry-run", "--format", "json", "--out", f.Name(), "--format", "progress"}, + files... + ) + + debug.Printf("Running `%s %s` for dry run", cmdName, strings.Join(dryRunArgs, " ")) + + output, err := exec.Command(cmdName, dryRunArgs...).CombinedOutput() + if err != nil { + return []plan.TestCase{}, fmt.Errorf("failed to run Cucumber dry run: %s", output) + } + + dryRunReport, parseErr := parseCucumberDryRunJSONOutput(f.Name()) // Use parser from cucumber_result_parser.go + if parseErr != nil { + return nil, fmt.Errorf("failed to parse cucumber dry run JSON report from %s: %w", f.Name(), parseErr) + } + + var testCases []plan.TestCase + for _, feature := range dryRunReport { + for _, scenario := range feature.Elements { + if scenario.Type == "scenario" { // Only include scenarios, not scenario outlines directly (examples are handled differently) + testCases = append(testCases, mapScenarioToTestCase(feature.URI, scenario)) + } else if scenario.Type == "scenario_outline" && scenario.Keyword == "Scenario Outline" { + // Scenario outlines themselves aren't runnable directly by path:line of the outline. + // Cucumber expands them into concrete scenarios based on their Examples tables. + // The JSON from a dry run might already include these expanded examples as individual 'scenario' type elements. + // If not, we'd need to parse scenario.Examples and generate test cases for each example row. + // For now, we assume the JSON includes expanded examples as type: "scenario". + // If the dry run JSON for outlines is different, this part needs adjustment. + // Let's log if we encounter an outline to see its structure. + debug.Printf("Encountered Scenario Outline: %s:%d. Its examples might be listed as separate scenarios.", feature.URI, scenario.Line) + } + } + } + + return testCases, nil +} + +// commandNameAndArgs replaces placeholders and returns command + args. +func (c Cucumber) commandNameAndArgs(cmd string, testCases []string) (string, []string, error) { + words, err := shellquote.Split(cmd) + if err != nil { + return "", []string{}, err + } + + idx := slices.Index(words, "{{testExamples}}") + if idx < 0 { + words = append(words, testCases...) + } else { + words = slices.Replace(words, idx, idx+1, testCases...) + } + + idx = slices.Index(words, "{{resultPath}}") + if idx >= 0 { + words = slices.Replace(words, idx, idx+1, c.ResultPath) + } + + return words[0], words[1:], nil +} + +// ---------------- Report parsing ------------------- +// ParseReport now uses CucumberFeature from cucumber_result_parser.go + +func (c Cucumber) ParseReport(path string) ([]CucumberFeature, error) { + var report []CucumberFeature + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("failed to read cucumber output: %v", err) + } + + if err := json.Unmarshal(data, &report); err != nil { + return nil, fmt.Errorf("failed to parse cucumber output: %s", err) + } + + return report, nil +} diff --git a/internal/runner/cucumber_result_parser.go b/internal/runner/cucumber_result_parser.go new file mode 100644 index 00000000..7fddeb2e --- /dev/null +++ b/internal/runner/cucumber_result_parser.go @@ -0,0 +1,146 @@ +//nolint:all +package runner + +import ( + "encoding/json" + "fmt" + "os" +) + +// CucumberFeature represents a single feature in Cucumber's JSON output. +type CucumberFeature struct { + URI string `json:"uri"` + ID string `json:"id"` + Keyword string `json:"keyword"` + Name string `json:"name"` + Description string `json:"description"` + Line int `json:"line"` + Elements []CucumberElement `json:"elements"` + Tags []CucumberTag `json:"tags,omitempty"` +} + +// CucumberElement represents a scenario or background in Cucumber's JSON output. +type CucumberElement struct { + ID string `json:"id"` + Keyword string `json:"keyword"` + Name string `json:"name"` + Description string `json:"description"` + Line int `json:"line"` + Type string `json:"type"` // e.g., "scenario", "background" + Steps []CucumberStep `json:"steps"` + Tags []CucumberTag `json:"tags,omitempty"` + // Examples []CucumberExample `json:"examples,omitempty"` // For scenario outlines +} + +// AggregatedStatus returns overall scenario status based on its steps. +// It mirrors the logic previously in cucumber.go but uses the parser's structs. +func (e CucumberElement) AggregatedStatus() string { + // If there's no result for a step (e.g. in a dry run for GetExamples), it shouldn't affect status. + // The primary use of AggregatedStatus is after a real run. + status := "passed" + for _, step := range e.Steps { + if step.Result == nil { + // A step with no result (e.g. from a dry run) shouldn't alter a 'passed' status + // unless other steps explicitly fail or are skipped. If all steps have no result, + // it's effectively passed from an aggregation perspective for determining if *any* step failed. + continue + } + switch step.Result.Status { + case "failed", "undefined", "errored": // 'errored' is a common status for unexpected issues + return "failed" + case "pending", "skipped": + // If a scenario has both skipped and passed steps, it's considered skipped. + // 'pending' takes precedence over 'passed' but 'failed' takes precedence over 'pending'. + if status != "failed" { // don't downgrade from failed + status = "pending" // Treat as pending/skipped + } + case "passed": + // If current status is 'passed', it remains 'passed'. + // If current status is 'pending', it remains 'pending'. + // No change needed here if step is passed. + default: + // Unknown status, treat cautiously, but for now, don't alter current aggregate status + // unless it's an explicit failure/skip. + } + } + return status +} + +// CucumberStep represents a single step in a scenario. +type CucumberStep struct { + Keyword string `json:"keyword"` + Name string `json:"name"` + Line int `json:"line"` + Result *CucumberResult `json:"result,omitempty"` + Match *CucumberMatch `json:"match,omitempty"` + DocString *CucumberDocString `json:"doc_string,omitempty"` + DataTableRows []CucumberDataTableRow `json:"rows,omitempty"` // For data tables +} + +// CucumberResult represents the result of a step execution. +type CucumberResult struct { + Status string `json:"status"` // e.g., "passed", "failed", "skipped", "undefined" + ErrorMessage string `json:"error_message,omitempty"` + Duration int64 `json:"duration,omitempty"` // Nanoseconds +} + +// CucumberMatch represents the matching step definition for a step. +type CucumberMatch struct { + Location string `json:"location"` // e.g., "features/step_definitions/steps.rb:5" +} + +// CucumberTag represents a tag in Cucumber. +type CucumberTag struct { + Name string `json:"name"` + Line int `json:"line"` +} + +// CucumberDocString represents a doc string argument for a step. +type CucumberDocString struct { + Value string `json:"value"` + ContentType string `json:"content_type"` + Line int `json:"line"` +} + +// CucumberDataTableRow represents a row in a step's data table. +type CucumberDataTableRow struct { + Cells []string `json:"cells"` +} + + +// ParseCucumberJSONReport parses the JSON output from a cucumber run (not dry run). +// This is for actual test results. +func ParseCucumberJSONReport(filePath string) ([]CucumberFeature, error) { + data, err := os.ReadFile(filePath) + if err != nil { + return nil, fmt.Errorf("failed to read cucumber json report file %s: %w", filePath, err) + } + + if len(data) == 0 { + // Empty file likely means no tests were run or an issue with output generation. + // Return an empty slice of features and no error, as this might be a valid state (e.g., no tests selected). + return []CucumberFeature{}, nil + } + + var features []CucumberFeature + err = json.Unmarshal(data, &features) + if err != nil { + // Attempt to unmarshal into a single feature object if the top level isn't an array + // Some cucumber versions might output a single feature object if only one feature file is processed. + var singleFeature CucumberFeature + if singleErr := json.Unmarshal(data, &singleFeature); singleErr == nil { + features = []CucumberFeature{singleFeature} + } else { + return nil, fmt.Errorf("failed to unmarshal cucumber json report from %s: %w. Single feature unmarshal error: %v", filePath, err, singleErr) + } + } + return features, nil +} + +// This is the function GetExamples will call. +// It's identical to ParseCucumberJSONReport for now, as the dry-run JSON structure +// for features and elements (scenarios) should be compatible. +// If dry-run JSON differs significantly for listing purposes, this function can be specialized. +func parseCucumberDryRunJSONOutput(filePath string) ([]CucumberFeature, error) { + return ParseCucumberJSONReport(filePath) +} diff --git a/internal/runner/cucumber_test.go b/internal/runner/cucumber_test.go new file mode 100644 index 00000000..2bb16eef --- /dev/null +++ b/internal/runner/cucumber_test.go @@ -0,0 +1,555 @@ +package runner + +import ( + "errors" + "os" + "sort" + "testing" + + "github.com/buildkite/test-engine-client/internal/plan" + "github.com/google/go-cmp/cmp" + "github.com/kballard/go-shellquote" +) + +func TestNewCucumber(t *testing.T) { + cases := []struct { + input RunnerConfig + want RunnerConfig + }{ + // default + { + input: RunnerConfig{}, + want: RunnerConfig{ + TestCommand: "cucumber --format pretty --format json --out {{resultPath}} {{testExamples}}", + TestFilePattern: "features/**/*.feature", + TestFileExcludePattern: "", + RetryTestCommand: "cucumber --format pretty --format json --out {{resultPath}} {{testExamples}}", + }, + }, + // custom + { + input: RunnerConfig{ + TestCommand: "cucumber --format json --out {{resultPath}} {{testExamples}}", + TestFilePattern: "features/api/**/*.feature", + TestFileExcludePattern: "features/experimental", + RetryTestCommand: "cucumber --format json --out {{resultPath}} {{testExamples}}", + }, + want: RunnerConfig{ + TestCommand: "cucumber --format json --out {{resultPath}} {{testExamples}}", + TestFilePattern: "features/api/**/*.feature", + TestFileExcludePattern: "features/experimental", + RetryTestCommand: "cucumber --format json --out {{resultPath}} {{testExamples}}", + }, + }, + } + + for _, c := range cases { + got := NewCucumber(c.input) + if diff := cmp.Diff(got.RunnerConfig, c.want); diff != "" { + t.Errorf("NewCucumber(%v) diff (-got +want):\n%s", c.input, diff) + } + } +} + +func TestCucumberRun(t *testing.T) { + changeCwd(t, "./testdata/cucumber") + + cucumber := NewCucumber(RunnerConfig{ + TestCommand: "cucumber --format json --out {{resultPath}}", + ResultPath: "cucumber.json", + }) + + t.Cleanup(func() { + os.RemoveAll("tmp") // Clean up the whole tmp directory + }) + + // Create the directory for the results + if err := os.MkdirAll("tmp", 0755); err != nil { + t.Fatalf("could not create tmp directory: %v", err) + } + + testCases := []plan.TestCase{ + {Path: "./features/spells/expelliarmus.feature"}, + } + result := NewRunResult([]plan.TestCase{}) + err := cucumber.Run(result, testCases, false) + + if err != nil { + t.Errorf("Cucumber.Run(%q) error = %v", testCases, err) + } + + if result.Status() != RunStatusPassed { + // Attempt to read and log the content of cucumber.json for debugging + jsonContent, readErr := os.ReadFile("tmp/cucumber.json") + if readErr != nil { + t.Logf("Failed to read tmp/cucumber.json: %v", readErr) + } else { + t.Logf("Content of tmp/cucumber.json:\n%s", string(jsonContent)) + } + t.Errorf("Cucumber.Run(%q) RunResult.Status = %v, want %v", testCases, result.Status(), RunStatusPassed) + } +} + +func TestCucumberRun_TestFailed(t *testing.T) { + changeCwd(t, "./testdata/cucumber") + + cucumber := NewCucumber(RunnerConfig{ + TestCommand: "cucumber --format json --out {{resultPath}}", + ResultPath: "tmp/cucumber.json", + }) + + t.Cleanup(func() { + os.RemoveAll("tmp") // Clean up the whole tmp directory + }) + + // Create the directory for the results + if err := os.MkdirAll("tmp", 0755); err != nil { + t.Fatalf("could not create tmp directory: %v", err) + } + + testCases := []plan.TestCase{ + {Path: "./features/failure.feature"}, + } + result := NewRunResult([]plan.TestCase{}) + err := cucumber.Run(result, testCases, false) + + if err != nil { + t.Errorf("Cucumber.Run(%q) error = %v", testCases, err) + } + + if result.Status() != RunStatusFailed { + t.Errorf("Cucumber.Run(%q) RunResult.Status = %v, want %v", testCases, result.Status(), RunStatusFailed) + } + + if len(result.FailedTests()) == 0 { + t.Errorf("Cucumber.Run(%q) expected failed tests but got none", testCases) + } +} + +func TestCucumberGetFiles(t *testing.T) { + cucumber := NewCucumber(RunnerConfig{ + TestFilePattern: "testdata/cucumber/features/**/*.feature", + }) + + got, err := cucumber.GetFiles() + if err != nil { + t.Errorf("Cucumber.GetFiles() error = %v", err) + } + + want := []string{ + "testdata/cucumber/features/another_feature.feature", + "testdata/cucumber/features/failure.feature", + "testdata/cucumber/features/simple_scenarios.feature", + "testdata/cucumber/features/spells/expelliarmus.feature", + } + + sort.Strings(got) + sort.Strings(want) + + if diff := cmp.Diff(got, want); diff != "" { + t.Errorf("Cucumber.GetFiles() diff (-got +want):\n%s", diff) + } +} + +func TestCucumberGetExamples(t *testing.T) { + changeCwd(t, "./testdata/cucumber") + + // Configure the Cucumber runner. + // c.Dir will be used as the CWD for the cucumber command by GetExamples. + // Feature file paths passed to GetExamples should be relative to this Dir. + cucumber := NewCucumber(RunnerConfig{ + TestCommand: "cucumber", // Base command; GetExamples adds necessary formatters. + }) + + files := []string{ + "features/simple_scenarios.feature", + "features/another_feature.feature", + } + + got, err := cucumber.GetExamples(files) + if err != nil { + t.Fatalf("Cucumber.GetExamples(%v) error = %v", files, err) + } + + want := []plan.TestCase{ + { + Path: "features/simple_scenarios.feature:5", + Name: "First simple scenario", + Identifier: "features/simple_scenarios.feature:5", + }, + { + Path: "features/simple_scenarios.feature:11", + Name: "Second simple scenario", + Identifier: "features/simple_scenarios.feature:11", + }, + { + Path: "features/simple_scenarios.feature:15", + Name: "A pending scenario", + Identifier: "features/simple_scenarios.feature:15", + }, + { + Path: "features/simple_scenarios.feature:19", + Name: "A skipped scenario", + Identifier: "features/simple_scenarios.feature:19", + }, + { + Path: "features/simple_scenarios.feature:23", + Name: "A failing scenario", + Identifier: "features/simple_scenarios.feature:23", + }, + { + Path: "features/another_feature.feature:3", + Name: "Scenario in another feature", + Identifier: "features/another_feature.feature:3", + }, + } + + // Sort slices for stable comparison, as order from GetExamples might not be guaranteed. + sort.Slice(got, func(i, j int) bool { return got[i].Path < got[j].Path }) + sort.Slice(want, func(i, j int) bool { return want[i].Path < want[j].Path }) + + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("Cucumber.GetExamples() diff (-want +got):\n%s", diff) + } + + // Test with no files provided + gotEmpty, errEmpty := cucumber.GetExamples([]string{}) + if errEmpty != nil { + t.Fatalf("Cucumber.GetExamples([]string{}) error = %v", errEmpty) + } + if len(gotEmpty) != 0 { + t.Errorf("Cucumber.GetExamples([]string{}) got %d examples, want 0", len(gotEmpty)) + } + + // Test with a feature file that contains no scenarios. + // Create a temporary empty feature file for this purpose. + emptyFeatureFilePath := "features/empty_for_test.feature" + f, err := os.Create(emptyFeatureFilePath) + if err != nil { + t.Fatalf("Failed to create empty feature file %s: %v", emptyFeatureFilePath, err) + } + f.Close() // Close the file immediately after creation. + defer os.Remove(emptyFeatureFilePath) // Clean up the empty file. + + // Path to GetExamples is relative to c.Dir + gotFromEmptyFeature, errFromEmptyFeature := cucumber.GetExamples([]string{"features/empty_for_test.feature"}) + // Cucumber's --dry-run might exit with a non-zero status if a feature file is empty or has no scenarios, + // which GetExamples would then report as an error. + // Or, it might exit successfully and produce an empty JSON array. + // If an error occurs, we log it. If scenarios are returned, it's a test failure. + if errFromEmptyFeature != nil { + t.Logf("Cucumber.GetExamples with an empty feature file returned an error (this may be expected behavior from cucumber CLI): %v", errFromEmptyFeature) + } + if len(gotFromEmptyFeature) != 0 { + t.Errorf("Cucumber.GetExamples with an empty feature file: got %d examples, want 0. Error (if any): %v", len(gotFromEmptyFeature), errFromEmptyFeature) + } +} + +func TestCucumberRun_IndividualScenarios(t *testing.T) { + changeCwd(t, "./testdata/cucumber") + + cucumber := NewCucumber(RunnerConfig{ + TestCommand: "cucumber --format json --out {{resultPath}} {{testExamples}}", + ResultPath: "tmp/cucumber_individual.json", + }) + + t.Cleanup(func() { + os.RemoveAll("tmp") // Clean up the whole tmp directory + }) + + // Create the directory for the results + if err := os.MkdirAll("tmp", 0755); err != nil { + t.Fatalf("could not create tmp directory: %v", err) + } + + // Define the subset of scenarios to run + // These identifiers must match what GetExamples produces and what Run uses for recording. + individualTestCases := []plan.TestCase{ + { + Path: "features/simple_scenarios.feature:5", + Name: "First simple scenario", + Identifier: "features/simple_scenarios.feature:5", + }, + { + Path: "features/another_feature.feature:3", + Name: "Scenario in another feature", + Identifier: "features/another_feature.feature:3", + }, + } + + result := NewRunResult([]plan.TestCase{}) // No muted tests for this test + err := cucumber.Run(result, individualTestCases, false) + + if err != nil { + t.Errorf("Cucumber.Run() with individual scenarios error = %v", err) + } + + if result.Status() != RunStatusPassed { + jsonContent, readErr := os.ReadFile(cucumber.ResultPath) + if readErr != nil { + t.Logf("Failed to read %s: %v", cucumber.ResultPath, readErr) + } else { + t.Logf("Content of %s:\n%s", cucumber.ResultPath, string(jsonContent)) + } + t.Errorf("Cucumber.Run() with individual scenarios RunResult.Status = %v, want %v", result.Status(), RunStatusPassed) + } + + // Verify that only the specified tests were run and passed + stats := result.Statistics() + if stats.Total != len(individualTestCases) { + t.Errorf("Expected %d total tests, got %d", len(individualTestCases), stats.Total) + } + if stats.PassedOnFirstRun != len(individualTestCases) { + t.Errorf("Expected %d passed tests, got %d", len(individualTestCases), stats.PassedOnFirstRun) + } + if stats.Failed != 0 { + t.Errorf("Expected 0 failed tests, got %d", stats.Failed) + } + if stats.Skipped != 0 { + t.Errorf("Expected 0 skipped tests, got %d", stats.Skipped) + } +} + +func TestCucumberRun_ScenarioStatuses(t *testing.T) { + changeCwd(t, "./testdata/cucumber") + + cucumberRunner := NewCucumber(RunnerConfig{ + TestCommand: "cucumber --format json --out {{resultPath}}", // {{testExamples}} will be appended by the runner if not present + ResultPath: "tmp/cucumber_statuses.json", + }) + + t.Cleanup(func() { + os.RemoveAll("tmp") + }) + + if err := os.MkdirAll("tmp", 0755); err != nil { + t.Fatalf("could not create tmp directory: %v", err) + } + + // 1. Discover scenarios from simple_scenarios.feature + featureFiles := []string{"features/simple_scenarios.feature"} + discoveredScenarios, err := cucumberRunner.GetExamples(featureFiles) + if err != nil { + t.Fatalf("Cucumber.GetExamples(%v) error = %v", featureFiles, err) + } + + expectedScenarioCount := 5 // 2 passing, 1 pending, 1 skipped, 1 failing + if len(discoveredScenarios) != expectedScenarioCount { + t.Errorf("Expected %d scenarios from GetExamples, got %d", expectedScenarioCount, len(discoveredScenarios)) + for i, sc := range discoveredScenarios { + t.Logf("Discovered scenario %d: ID=%s, Name=%s, Path=%s", i, sc.Identifier, sc.Name, sc.Path) + } + } + + // 2. Run all discovered scenarios + result := NewRunResult([]plan.TestCase{}) // No muted tests + // runErr will capture errors from the cucumber command execution itself (e.g., command not found, non-zero exit for reasons other than test failures) + // Test failures themselves are recorded in the result object and don't necessarily cause cucumberRunner.Run to return an error. + _ = cucumberRunner.Run(result, discoveredScenarios, false) // We expect test failures, so the error from Run() might be misleading here if it only reflects cucumber's exit code. + + // 3. Assert RunResult status and statistics + if result.Status() != RunStatusFailed { + // Log the JSON output if the overall status is not what we expect. + jsonContent, readErr := os.ReadFile(cucumberRunner.ResultPath) + if readErr != nil { + t.Logf("Failed to read result file %s: %v", cucumberRunner.ResultPath, readErr) + } else { + t.Logf("Content of result file %s:\n%s", cucumberRunner.ResultPath, string(jsonContent)) + } + t.Errorf("RunResult.Status() = %v, want %v", result.Status(), RunStatusFailed) + } + + stats := result.Statistics() + if stats.Total != expectedScenarioCount { + t.Errorf("Statistics: Total = %d, want %d", stats.Total, expectedScenarioCount) + } + + expectedPassed := 2 + expectedFailed := 1 + expectedSkipped := 2 // Pending maps to skipped + + if stats.PassedOnFirstRun != expectedPassed { + t.Errorf("Statistics: PassedOnFirstRun = %d, want %d", stats.PassedOnFirstRun, expectedPassed) + } + if stats.Failed != expectedFailed { + t.Errorf("Statistics: Failed = %d, want %d", stats.Failed, expectedFailed) + } + if stats.Skipped != expectedSkipped { + t.Errorf("Statistics: Skipped = %d, want %d", stats.Skipped, expectedSkipped) + } + + // 4. Optional: Verify status of specific scenarios by identifier + // Ensure test identifiers match exactly what GetExamples produces and Run uses. + // These are based on the line numbers in simple_scenarios.feature + // Keys are Scope/Name/Path as stored by RunResult + wantStatuses := map[string]TestStatus{ // Use TestStatus from runner package + "Simple Scenarios/First simple scenario/features/simple_scenarios.feature:5": TestStatusPassed, + "Simple Scenarios/Second simple scenario/features/simple_scenarios.feature:11": TestStatusPassed, + "Simple Scenarios/A pending scenario/features/simple_scenarios.feature:15": TestStatusSkipped, // Pending scenario + "Simple Scenarios/A skipped scenario/features/simple_scenarios.feature:19": TestStatusSkipped, // Skipped scenario + "Simple Scenarios/A failing scenario/features/simple_scenarios.feature:23": TestStatusFailed, // Failing scenario + } + + if len(result.tests) != expectedScenarioCount { // Access result.tests map directly + t.Errorf("Expected %d test results, got %d", expectedScenarioCount, len(result.tests)) + } + + for identifier, testCaseResult := range result.tests { // Iterate over map + expectedStatus, ok := wantStatuses[identifier] // Use identifier from map key + if !ok { + t.Errorf("Unexpected test case identifier in results: %s", identifier) + continue + } + if testCaseResult.Status != expectedStatus { + t.Errorf("Status for %s: got %v, want %v", identifier, testCaseResult.Status, expectedStatus) + } + } +} + +func TestCucumberCommandNameAndArgs_WithInterpolationPlaceholder(t *testing.T) { + testCases := []string{"features/spells/expelliarmus.feature", "features/failure.feature"} + testCommand := "cucumber --format json --out {{resultPath}} {{testExamples}}" + + c := NewCucumber(RunnerConfig{ + TestCommand: testCommand, + ResultPath: "cucumber.json", + }) + + gotName, gotArgs, err := c.commandNameAndArgs(testCommand, testCases) + if err != nil { + t.Errorf("commandNameAndArgs(%q, %q) error = %v", testCases, testCommand, err) + } + + wantName := "cucumber" + wantArgs := []string{"--format", "json", "--out", "cucumber.json", "features/spells/expelliarmus.feature", "features/failure.feature"} + + if diff := cmp.Diff(gotName, wantName); diff != "" { + t.Errorf("commandNameAndArgs(%q, %q) diff (-got +want):\n%s", testCases, testCommand, diff) + } + if diff := cmp.Diff(gotArgs, wantArgs); diff != "" { + t.Errorf("commandNameAndArgs(%q, %q) diff (-got +want):\n%s", testCases, testCommand, diff) + } +} + +func TestCucumberCommandNameAndArgs_WithoutTestExamplesPlaceholder(t *testing.T) { + testCases := []string{"features/spells/expelliarmus.feature", "features/failure.feature"} + testCommand := "cucumber --format json --out {{resultPath}}" + + c := NewCucumber(RunnerConfig{ + TestCommand: testCommand, + ResultPath: "cucumber.json", + }) + + gotName, gotArgs, err := c.commandNameAndArgs(testCommand, testCases) + if err != nil { + t.Errorf("commandNameAndArgs(%q, %q) error = %v", testCases, testCommand, err) + } + + wantName := "cucumber" + wantArgs := []string{"--format", "json", "--out", "cucumber.json", "features/spells/expelliarmus.feature", "features/failure.feature"} + + if diff := cmp.Diff(gotName, wantName); diff != "" { + t.Errorf("commandNameAndArgs(%q, %q) diff (-got +want):\n%s", testCases, testCommand, diff) + } + if diff := cmp.Diff(gotArgs, wantArgs); diff != "" { + t.Errorf("commandNameAndArgs(%q, %q) diff (-got +want):\n%s", testCases, testCommand, diff) + } +} + +func TestCucumberCommandNameAndArgs_InvalidTestCommand(t *testing.T) { + testCases := []string{"features/spells/expelliarmus.feature", "features/failure.feature"} + testCommand := "cucumber --format json --out '{{resultPath}} {{testExamples}}" + + c := NewCucumber(RunnerConfig{ + TestCommand: testCommand, + }) + + gotName, gotArgs, err := c.commandNameAndArgs(testCommand, testCases) + + wantName := "" + wantArgs := []string{} + + if diff := cmp.Diff(gotName, wantName); diff != "" { + t.Errorf("commandNameAndArgs() diff (-got +want):\n%s", diff) + } + if diff := cmp.Diff(gotArgs, wantArgs); diff != "" { + t.Errorf("commandNameAndArgs() diff (-got +want):\n%s", diff) + } + if !errors.Is(err, shellquote.UnterminatedSingleQuoteError) { + t.Errorf("commandNameAndArgs() error = %v, want %v", err, shellquote.UnterminatedSingleQuoteError) + } +} + +func TestCucumberGetExamples_WithOtherFormatters(t *testing.T) { + changeCwd(t, "./testdata/cucumber") + + files := []string{"features/simple_scenarios.feature"} + want := []plan.TestCase{ + { + Path: "features/simple_scenarios.feature:5", + Name: "First simple scenario", + Identifier: "features/simple_scenarios.feature:5", + }, + { + Path: "features/simple_scenarios.feature:11", + Name: "Second simple scenario", + Identifier: "features/simple_scenarios.feature:11", + }, + { + Path: "features/simple_scenarios.feature:15", + Name: "A pending scenario", + Identifier: "features/simple_scenarios.feature:15", + }, + { + Path: "features/simple_scenarios.feature:19", + Name: "A skipped scenario", + Identifier: "features/simple_scenarios.feature:19", + }, + { + Path: "features/simple_scenarios.feature:23", + Name: "A failing scenario", + Identifier: "features/simple_scenarios.feature:23", + }, + } + sort.Slice(want, func(i, j int) bool { return want[i].Path < want[j].Path }) + + // Create a temporary file to store the JSON output of the cucumber dry run for one of the commands. + // So we don't end up with a lot of files after running this test. + // We'll clean up the file after the test. + // Ensure tmp directory exists (it's created by other tests, but good for standalone robustness) + if err := os.MkdirAll("tmp", 0755); err != nil { + t.Fatalf("could not create tmp directory: %v", err) + } + f, err := os.CreateTemp("tmp", "cucumber-*.html") + if err != nil { + t.Fatalf("os.CreateTemp() error = %v", err) + } + f.Close() + // The global Cleanup in TestMain or individual test Cleanups for "tmp" should handle this. + // os.Remove(f.Name()) // Not strictly needed if tmp is cleaned up globally. + + commands := []string{ + "cucumber --format progress", + "cucumber --format pretty", + "cucumber --format html --out " + f.Name(), + "cucumber --format progress --format json --out some_other.json", // GetExamples should override this json output for dry-run + } + + for _, command := range commands { + cucumberRunner := NewCucumber(RunnerConfig{ // Renamed to avoid conflict with package name + TestCommand: command, + }) + t.Run(command, func(t *testing.T) { + got, err := cucumberRunner.GetExamples(files) + if err != nil { + t.Fatalf("Cucumber.GetExamples(%v) with TestCommand '%s' error = %v", files, command, err) + } + + sort.Slice(got, func(i, j int) bool { return got[i].Path < got[j].Path }) + + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("Cucumber.GetExamples(%v) with TestCommand '%s' diff (-want +got):\n%s", files, command, diff) + } + }) + } +} diff --git a/internal/runner/detector.go b/internal/runner/detector.go index cc4c93a9..38815c5f 100644 --- a/internal/runner/detector.go +++ b/internal/runner/detector.go @@ -51,8 +51,10 @@ func DetectRunner(cfg config.Config) (TestRunner, error) { return NewPytestPants(runnerConfig), nil case "gotest": return NewGoTest(runnerConfig), nil + case "cucumber": + return NewCucumber(runnerConfig), nil default: // Update the error message to include the new runner - return nil, errors.New("runner value is invalid, possible values are 'rspec', 'jest', 'cypress', 'playwright', 'pytest', 'pytest-pants', or 'gotest'") + return nil, errors.New("runner value is invalid, possible values are 'rspec', 'jest', 'cypress', 'playwright', 'pytest', 'pytest-pants', 'gotest', or 'cucumber'") } } diff --git a/internal/runner/doc.go b/internal/runner/doc.go index dd831062..53f2461b 100644 --- a/internal/runner/doc.go +++ b/internal/runner/doc.go @@ -1,2 +1,3 @@ // Package runner provides the test runners that run sets of test cases. +// Supported runners: rspec, jest, cypress, playwright, pytest, gotest, cucumber. package runner diff --git a/internal/runner/testdata/cucumber/Gemfile b/internal/runner/testdata/cucumber/Gemfile new file mode 100644 index 00000000..4778e38e --- /dev/null +++ b/internal/runner/testdata/cucumber/Gemfile @@ -0,0 +1,6 @@ +# frozen_string_literal: true + +source "https://rubygems.org" + +gem "base64" +gem "cucumber" diff --git a/internal/runner/testdata/cucumber/Gemfile.lock b/internal/runner/testdata/cucumber/Gemfile.lock new file mode 100644 index 00000000..9a58a31e --- /dev/null +++ b/internal/runner/testdata/cucumber/Gemfile.lock @@ -0,0 +1,47 @@ +GEM + remote: https://rubygems.org/ + specs: + base64 (0.3.0) + bigdecimal (3.2.0) + builder (3.3.0) + cucumber (9.2.1) + builder (~> 3.2) + cucumber-ci-environment (> 9, < 11) + cucumber-core (> 13, < 14) + cucumber-cucumber-expressions (~> 17.0) + cucumber-gherkin (> 24, < 28) + cucumber-html-formatter (> 20.3, < 22) + cucumber-messages (> 19, < 25) + diff-lcs (~> 1.5) + mini_mime (~> 1.1) + multi_test (~> 1.1) + sys-uname (~> 1.2) + cucumber-ci-environment (10.0.1) + cucumber-core (13.0.3) + cucumber-gherkin (>= 27, < 28) + cucumber-messages (>= 20, < 23) + cucumber-tag-expressions (> 5, < 7) + cucumber-cucumber-expressions (17.1.0) + bigdecimal + cucumber-gherkin (27.0.0) + cucumber-messages (>= 19.1.4, < 23) + cucumber-html-formatter (21.9.0) + cucumber-messages (> 19, < 28) + cucumber-messages (22.0.0) + cucumber-tag-expressions (6.1.2) + diff-lcs (1.6.2) + ffi (1.17.2-arm64-darwin) + mini_mime (1.1.5) + multi_test (1.1.0) + sys-uname (1.3.1) + ffi (~> 1.1) + +PLATFORMS + arm64-darwin + +DEPENDENCIES + base64 + cucumber + +BUNDLED WITH + 2.6.7 diff --git a/internal/runner/testdata/cucumber/features/another_feature.feature b/internal/runner/testdata/cucumber/features/another_feature.feature new file mode 100644 index 00000000..152ff783 --- /dev/null +++ b/internal/runner/testdata/cucumber/features/another_feature.feature @@ -0,0 +1,5 @@ +Feature: Another Feature + + Scenario: Scenario in another feature + # Line 4 + Given a step here diff --git a/internal/runner/testdata/cucumber/features/failure.feature b/internal/runner/testdata/cucumber/features/failure.feature new file mode 100644 index 00000000..7f3fdb76 --- /dev/null +++ b/internal/runner/testdata/cucumber/features/failure.feature @@ -0,0 +1,4 @@ +Feature: Failing Test + Scenario: This scenario is supposed to fail + Given A functioning wand + Then Something unexpected happens that causes a failure diff --git a/internal/runner/testdata/cucumber/features/simple_scenarios.feature b/internal/runner/testdata/cucumber/features/simple_scenarios.feature new file mode 100644 index 00000000..9db1a71b --- /dev/null +++ b/internal/runner/testdata/cucumber/features/simple_scenarios.feature @@ -0,0 +1,25 @@ +Feature: Simple Scenarios + Background: + Given a background step + + Scenario: First simple scenario + # Line 5 + Given a step + When another step + Then a final step + + Scenario: Second simple scenario + # Line 10 + Given a different step + + Scenario: A pending scenario + # Line 15 + Given a step that marks as pending + + Scenario: A skipped scenario + # Line 19 + Given a step that skips + + Scenario: A failing scenario + # Line 23 + Given a step that fails diff --git a/internal/runner/testdata/cucumber/features/spells/expelliarmus.feature b/internal/runner/testdata/cucumber/features/spells/expelliarmus.feature new file mode 100644 index 00000000..32bbc0a5 --- /dev/null +++ b/internal/runner/testdata/cucumber/features/spells/expelliarmus.feature @@ -0,0 +1,4 @@ +Feature: Expelliarmus + Scenario: Disarms the opponent + Given A functioning wand + Then Opponent is disarmed diff --git a/internal/runner/testdata/cucumber/features/step_definitions/steps.rb b/internal/runner/testdata/cucumber/features/step_definitions/steps.rb new file mode 100644 index 00000000..ab8d2d08 --- /dev/null +++ b/internal/runner/testdata/cucumber/features/step_definitions/steps.rb @@ -0,0 +1,51 @@ +Given("A functioning wand") do + true +end + +Then('the result should be {int}') do |int| + expect(@result).to eq(int) +end + +Given('a background step') do + true +end + +Given('a step') do + true +end + +When('another step') do + true +end + +Then('a final step') do + true +end + +Given('a different step') do + true +end + +Given('a step here') do + true +end + +Then("Opponent is disarmed") do + true +end + +Then("Something unexpected happens that causes a failure") do + raise "This step is designed to fail!" +end + +Given('a step that marks as pending') do + pending "This step is intentionally pending." +end + +Given('a step that skips') do + skip_this_scenario("This scenario is intentionally skipped.") +end + +Given('a step that fails') do + raise "This step is designed to fail!" +end diff --git a/main.go b/main.go index 080bd663..9545a7d1 100644 --- a/main.go +++ b/main.go @@ -392,8 +392,8 @@ func createRequestParam(ctx context.Context, cfg config.Config, files []string, } // Splitting files by example is only supported for rspec runner. - if runner.Name() != "RSpec" { - params := api.TestPlanParams{ + if runner.Name() != "RSpec" && runner.Name() != "Cucumber" { + return api.TestPlanParams{ Identifier: cfg.Identifier, Parallelism: cfg.Parallelism, Branch: cfg.Branch, From de236ab30387008b34407057ab2554af3b4a1101 Mon Sep 17 00:00:00 2001 From: Stephen Bell Date: Tue, 17 Jun 2025 09:29:51 +1200 Subject: [PATCH 2/4] Lint --- internal/runner/cucumber.go | 2 +- internal/runner/cucumber_result_parser.go | 29 +++++++++++------------ internal/runner/cucumber_test.go | 12 +++++----- 3 files changed, 21 insertions(+), 22 deletions(-) diff --git a/internal/runner/cucumber.go b/internal/runner/cucumber.go index db14fe6c..ce37b4c3 100644 --- a/internal/runner/cucumber.go +++ b/internal/runner/cucumber.go @@ -178,7 +178,7 @@ func (c Cucumber) GetExamples(files []string) ([]plan.TestCase, error) { dryRunArgs := append( []string{"--dry-run", "--format", "json", "--out", f.Name(), "--format", "progress"}, - files... + files..., ) debug.Printf("Running `%s %s` for dry run", cmdName, strings.Join(dryRunArgs, " ")) diff --git a/internal/runner/cucumber_result_parser.go b/internal/runner/cucumber_result_parser.go index 7fddeb2e..6200abd7 100644 --- a/internal/runner/cucumber_result_parser.go +++ b/internal/runner/cucumber_result_parser.go @@ -21,14 +21,14 @@ type CucumberFeature struct { // CucumberElement represents a scenario or background in Cucumber's JSON output. type CucumberElement struct { - ID string `json:"id"` - Keyword string `json:"keyword"` - Name string `json:"name"` - Description string `json:"description"` - Line int `json:"line"` - Type string `json:"type"` // e.g., "scenario", "background" - Steps []CucumberStep `json:"steps"` - Tags []CucumberTag `json:"tags,omitempty"` + ID string `json:"id"` + Keyword string `json:"keyword"` + Name string `json:"name"` + Description string `json:"description"` + Line int `json:"line"` + Type string `json:"type"` // e.g., "scenario", "background" + Steps []CucumberStep `json:"steps"` + Tags []CucumberTag `json:"tags,omitempty"` // Examples []CucumberExample `json:"examples,omitempty"` // For scenario outlines } @@ -68,12 +68,12 @@ func (e CucumberElement) AggregatedStatus() string { // CucumberStep represents a single step in a scenario. type CucumberStep struct { - Keyword string `json:"keyword"` - Name string `json:"name"` - Line int `json:"line"` - Result *CucumberResult `json:"result,omitempty"` - Match *CucumberMatch `json:"match,omitempty"` - DocString *CucumberDocString `json:"doc_string,omitempty"` + Keyword string `json:"keyword"` + Name string `json:"name"` + Line int `json:"line"` + Result *CucumberResult `json:"result,omitempty"` + Match *CucumberMatch `json:"match,omitempty"` + DocString *CucumberDocString `json:"doc_string,omitempty"` DataTableRows []CucumberDataTableRow `json:"rows,omitempty"` // For data tables } @@ -107,7 +107,6 @@ type CucumberDataTableRow struct { Cells []string `json:"cells"` } - // ParseCucumberJSONReport parses the JSON output from a cucumber run (not dry run). // This is for actual test results. func ParseCucumberJSONReport(filePath string) ([]CucumberFeature, error) { diff --git a/internal/runner/cucumber_test.go b/internal/runner/cucumber_test.go index 2bb16eef..945d4fd6 100644 --- a/internal/runner/cucumber_test.go +++ b/internal/runner/cucumber_test.go @@ -228,7 +228,7 @@ func TestCucumberGetExamples(t *testing.T) { if err != nil { t.Fatalf("Failed to create empty feature file %s: %v", emptyFeatureFilePath, err) } - f.Close() // Close the file immediately after creation. + f.Close() // Close the file immediately after creation. defer os.Remove(emptyFeatureFilePath) // Clean up the empty file. // Path to GetExamples is relative to c.Dir @@ -384,10 +384,10 @@ func TestCucumberRun_ScenarioStatuses(t *testing.T) { // Keys are Scope/Name/Path as stored by RunResult wantStatuses := map[string]TestStatus{ // Use TestStatus from runner package "Simple Scenarios/First simple scenario/features/simple_scenarios.feature:5": TestStatusPassed, - "Simple Scenarios/Second simple scenario/features/simple_scenarios.feature:11": TestStatusPassed, - "Simple Scenarios/A pending scenario/features/simple_scenarios.feature:15": TestStatusSkipped, // Pending scenario - "Simple Scenarios/A skipped scenario/features/simple_scenarios.feature:19": TestStatusSkipped, // Skipped scenario - "Simple Scenarios/A failing scenario/features/simple_scenarios.feature:23": TestStatusFailed, // Failing scenario + "Simple Scenarios/Second simple scenario/features/simple_scenarios.feature:11": TestStatusPassed, + "Simple Scenarios/A pending scenario/features/simple_scenarios.feature:15": TestStatusSkipped, // Pending scenario + "Simple Scenarios/A skipped scenario/features/simple_scenarios.feature:19": TestStatusSkipped, // Skipped scenario + "Simple Scenarios/A failing scenario/features/simple_scenarios.feature:23": TestStatusFailed, // Failing scenario } if len(result.tests) != expectedScenarioCount { // Access result.tests map directly @@ -520,7 +520,7 @@ func TestCucumberGetExamples_WithOtherFormatters(t *testing.T) { if err := os.MkdirAll("tmp", 0755); err != nil { t.Fatalf("could not create tmp directory: %v", err) } - f, err := os.CreateTemp("tmp", "cucumber-*.html") + f, err := os.CreateTemp("tmp", "cucumber-*.html") if err != nil { t.Fatalf("os.CreateTemp() error = %v", err) } From 26de99f19ee3e224dd6214d9706962e2420a329b Mon Sep 17 00:00:00 2001 From: Stephen Bell Date: Tue, 17 Jun 2025 09:36:24 +1200 Subject: [PATCH 3/4] Fix early return --- main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.go b/main.go index 9545a7d1..42779c51 100644 --- a/main.go +++ b/main.go @@ -393,7 +393,7 @@ func createRequestParam(ctx context.Context, cfg config.Config, files []string, // Splitting files by example is only supported for rspec runner. if runner.Name() != "RSpec" && runner.Name() != "Cucumber" { - return api.TestPlanParams{ + params := api.TestPlanParams{ Identifier: cfg.Identifier, Parallelism: cfg.Parallelism, Branch: cfg.Branch, From e883690ae35791daf2c9e9065eff1480db64ea58 Mon Sep 17 00:00:00 2001 From: Steve Bell Date: Thu, 19 Jun 2025 11:48:24 +1200 Subject: [PATCH 4/4] Update main.go Co-authored-by: Tom Meier --- main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.go b/main.go index 42779c51..a6edc1a7 100644 --- a/main.go +++ b/main.go @@ -391,7 +391,7 @@ func createRequestParam(ctx context.Context, cfg config.Config, files []string, }) } - // Splitting files by example is only supported for rspec runner. + // Splitting files by example is only supported for rspec runner & cucumber if runner.Name() != "RSpec" && runner.Name() != "Cucumber" { params := api.TestPlanParams{ Identifier: cfg.Identifier,