Codify runner feature support

internal/command/files.go

@@ -5,9 +5,11 @@ import (
 	"net/http"
 	"os"
 	"strings"
+
+	"github.com/buildkite/test-engine-client/internal/runner"
 )
 
-func getTestFiles(fileList string, testRunner TestRunner) ([]string, error) {
+func getTestFiles(fileList string, testRunner runner.TestRunner) ([]string, error) {
 	if fileList != "" {
 		return getTestFilesFromFile(fileList)
 	} else {

internal/command/request_param.go

@@ -8,6 +8,7 @@ import (
 	"github.com/buildkite/test-engine-client/internal/config"
 	"github.com/buildkite/test-engine-client/internal/debug"
 	"github.com/buildkite/test-engine-client/internal/plan"
+	"github.com/buildkite/test-engine-client/internal/runner"
 )
 
 // createRequestParam generates the parameters needed for a test plan request.
@@ -18,16 +19,16 @@ import (
 //
 // If tag filtering is enabled, all files are split into examples to support filtering.
 // Currently only the Pytest runner supports tag filtering.
-func createRequestParam(ctx context.Context, cfg *config.Config, files []string, client api.Client, runner TestRunner) (api.TestPlanParams, error) {
+func createRequestParam(ctx context.Context, cfg *config.Config, files []string, client api.Client, runner runner.TestRunner) (api.TestPlanParams, error) {
 	testFiles := []plan.TestCase{}
 	for _, file := range files {
 		testFiles = append(testFiles, plan.TestCase{
 			Path: file,
 		})
 	}
 
-	// Splitting files by example is only supported for rspec, cucumber, and pytest runners
-	if runner.Name() != "RSpec" && runner.Name() != "Cucumber" && runner.Name() != "pytest" {
+	// Short circuit here if the runner doesn't support split by example
+	if !runner.SupportedFeatures().SplitByExample {
 		params := api.TestPlanParams{
 			Identifier:     cfg.Identifier,
 			Parallelism:    cfg.Parallelism,
@@ -86,7 +87,7 @@ func createRequestParam(ctx context.Context, cfg *config.Config, files []string,
 }
 
 // Splits all the test files into examples to support tag filtering.
-func splitAllFiles(files []plan.TestCase, runner TestRunner) (api.TestPlanParamsTest, error) {
+func splitAllFiles(files []plan.TestCase, runner runner.TestRunner) (api.TestPlanParamsTest, error) {
 	debug.Printf("Splitting all %d files", len(files))
 	filePaths := make([]string, 0, len(files))
 	for _, file := range files {
@@ -108,7 +109,7 @@ func splitAllFiles(files []plan.TestCase, runner TestRunner) (api.TestPlanParams
 // filterAndSplitFiles filters the test files through the Test Engine API and splits the filtered files into examples.
 // It returns the test plan parameters with the examples from the filtered files and the remaining files.
 // An error is returned if there is a failure in any of the process.
-func filterAndSplitFiles(ctx context.Context, cfg *config.Config, client api.Client, files []plan.TestCase, runner TestRunner) (api.TestPlanParamsTest, error) {
+func filterAndSplitFiles(ctx context.Context, cfg *config.Config, client api.Client, files []plan.TestCase, runner runner.TestRunner) (api.TestPlanParamsTest, error) {
 	// Filter files that need to be split.
 	debug.Printf("Filtering %d files", len(files))
 	filteredFiles, err := client.FilterTests(ctx, cfg.SuiteSlug, api.FilterTestsParams{

internal/command/run.go

@@ -19,19 +19,6 @@ import (
 	"github.com/olekukonko/tablewriter"
 )
 
-type TestRunner interface {
-	// Run takes testCases as input, executes the test against the test cases, and mutates the runner.RunResult with the test results.
-	Run(result *runner.RunResult, testCases []plan.TestCase, retry bool) error
-	// GetExamples discovers all tests within given files.
-	// This function is only used for split by example use case. Currently only supported by RSpec.
-	GetExamples(files []string) ([]plan.TestCase, error)
-	// GetFiles discover all test files that the runner should execute.
-	// This is sent to server-side when creating test plan.
-	// This is also used to obtain a fallback non-intelligent test splitting mechanism.
-	GetFiles() ([]string, error)
-	Name() string
-}
-
 const Logo = `
 ______ ______ _____
 ___  /____  /___  /____________
@@ -206,7 +193,7 @@ func sendMetadata(ctx context.Context, apiClient *api.Client, cfg *config.Config
 // For next reader, there is a small caveat with current implementation:
 // - testCases and timeline are both expected to be mutated.
 // - testCases in this case serve both as input and output -> we should probably change it.
-func runTestsWithRetry(testRunner TestRunner, testsCases *[]plan.TestCase, maxRetries int, mutedTests []plan.TestCase, timeline *[]api.Timeline, retryForMutedTest bool, failOnNoTests bool) (runner.RunResult, error) {
+func runTestsWithRetry(testRunner runner.TestRunner, testsCases *[]plan.TestCase, maxRetries int, mutedTests []plan.TestCase, timeline *[]api.Timeline, retryForMutedTest bool, failOnNoTests bool) (runner.RunResult, error) {
 	attemptCount := 0
 
 	// Create a new run result with muted tests to keep track of the results.
@@ -293,7 +280,7 @@ func logSignalAndExit(name string, signal syscall.Signal) {
 
 // fetchOrCreateTestPlan fetches a test plan from the server, or creates a
 // fallback plan if the server is unavailable or returns an error plan.
-func fetchOrCreateTestPlan(ctx context.Context, apiClient *api.Client, cfg *config.Config, files []string, testRunner TestRunner) (plan.TestPlan, error) {
+func fetchOrCreateTestPlan(ctx context.Context, apiClient *api.Client, cfg *config.Config, files []string, testRunner runner.TestRunner) (plan.TestPlan, error) {
 	debug.Println("Fetching test plan")
 
 	// Fetch the plan from the server's cache.

internal/command/run_test.go

@@ -752,7 +752,7 @@ func TestCreateRequestParams_NonRSpec(t *testing.T) {
 	}))
 	defer svr.Close()
 
-	runners := []TestRunner{
+	runners := []runner.TestRunner{
 		runner.Jest{}, runner.Playwright{}, runner.Cypress{},
 	}
 

internal/runner/cucumber.go

@@ -45,6 +45,17 @@ func NewCucumber(c RunnerConfig) Cucumber {
 	}
 }
 
+func (c Cucumber) SupportedFeatures() SupportedFeatures {
+	return SupportedFeatures{
+		SplitByFile:     true,
+		SplitByExample:  true,
+		FilterTestFiles: true,
+		AutoRetry:       true,
+		Mute:            true,
+		Skip:            true,
+	}
+}
+
 func (c Cucumber) Name() string {
 	return "Cucumber"
 }

internal/runner/custom.go

@@ -33,6 +33,17 @@ func NewCustom(r RunnerConfig) (Custom, error) {
 	}, nil
 }
 
+func (c Custom) SupportedFeatures() SupportedFeatures {
+	return SupportedFeatures{
+		SplitByFile:     true,
+		SplitByExample:  false,
+		FilterTestFiles: true,
+		AutoRetry:       false,
+		Mute:            true,
+		Skip:            false,
+	}
+}
+
 func (r Custom) Name() string {
 	return "Custom test runner"
 }

internal/runner/cypress.go

@@ -33,6 +33,17 @@ func NewCypress(c RunnerConfig) Cypress {
 	}
 }
 
+func (c Cypress) SupportedFeatures() SupportedFeatures {
+	return SupportedFeatures{
+		SplitByFile:     true,
+		SplitByExample:  false,
+		FilterTestFiles: true,
+		AutoRetry:       false,
+		Mute:            false,
+		Skip:            false,
+	}
+}
+
 func (c Cypress) Run(result *RunResult, testCases []plan.TestCase, retry bool) error {
 	testPaths := make([]string, len(testCases))
 	for i, tc := range testCases {

internal/runner/detector.go

@@ -4,29 +4,8 @@ import (
 	"fmt"
 
 	"github.com/buildkite/test-engine-client/internal/config"
-	"github.com/buildkite/test-engine-client/internal/plan"
 )
 
-type RunnerConfig struct {
-	TestRunner             string
-	TestCommand            string
-	TestFilePattern        string
-	TestFileExcludePattern string
-	RetryTestCommand       string
-	TagFilters             string
-	// ResultPath is used internally so bktec can read result from Test Runner.
-	// User typically don't need to worry about setting this except in in RSpec and playwright.
-	// In playwright, for example, it can only be configured via a config file, therefore it's mandatory for user to set.
-	ResultPath string
-}
-
-type TestRunner interface {
-	Run(result *RunResult, testCases []plan.TestCase, retry bool) error
-	GetExamples(files []string) ([]plan.TestCase, error)
-	GetFiles() ([]string, error)
-	Name() string
-}
-
 func DetectRunner(cfg *config.Config) (TestRunner, error) {
 	runnerConfig := RunnerConfig{
 		TestRunner:             cfg.TestRunner,

internal/runner/gotest.go

@@ -32,6 +32,17 @@ func NewGoTest(c RunnerConfig) GoTest {
 	}
 }
 
+func (g GoTest) SupportedFeatures() SupportedFeatures {
+	return SupportedFeatures{
+		SplitByFile:     false,
+		SplitByExample:  false,
+		FilterTestFiles: false,
+		AutoRetry:       true,
+		Mute:            true,
+		Skip:            false,
+	}
+}
+
 func (g GoTest) Name() string {
 	return "gotest"
 }

internal/runner/jest.go

@@ -38,6 +38,17 @@ func NewJest(j RunnerConfig) Jest {
 	}
 }
 
+func (j Jest) SupportedFeatures() SupportedFeatures {
+	return SupportedFeatures{
+		SplitByFile:     true,
+		SplitByExample:  false,
+		FilterTestFiles: true,
+		AutoRetry:       true,
+		Mute:            true,
+		Skip:            false,
+	}
+}
+
 func (j Jest) Name() string {
 	return "Jest"
 }

internal/runner/playwright.go

@@ -35,6 +35,17 @@ func NewPlaywright(p RunnerConfig) Playwright {
 	}
 }
 
+func (p Playwright) SupportedFeatures() SupportedFeatures {
+	return SupportedFeatures{
+		SplitByFile:     true,
+		SplitByExample:  false,
+		FilterTestFiles: true,
+		AutoRetry:       true,
+		Mute:            true,
+		Skip:            false,
+	}
+}
+
 func (p Playwright) Run(result *RunResult, testCases []plan.TestCase, retry bool) error {
 	testPaths := make([]string, len(testCases))
 	for i, tc := range testCases {

internal/runner/pytest.go

@@ -59,6 +59,17 @@ func NewPytest(c RunnerConfig) Pytest {
 	}
 }
 
+func (p Pytest) SupportedFeatures() SupportedFeatures {
+	return SupportedFeatures{
+		SplitByFile:     true,
+		SplitByExample:  true,
+		FilterTestFiles: true,
+		AutoRetry:       true,
+		Mute:            true,
+		Skip:            false,
+	}
+}
+
 func (p Pytest) Run(result *RunResult, testCases []plan.TestCase, retry bool) error {
 	testPaths := make([]string, len(testCases))
 	for i, tc := range testCases {

internal/runner/pytest_pants.go

@@ -50,6 +50,17 @@ func NewPytestPants(c RunnerConfig) PytestPants {
 	}
 }
 
+func (p PytestPants) SupportedFeatures() SupportedFeatures {
+	return SupportedFeatures{
+		SplitByFile:     false,
+		SplitByExample:  false,
+		FilterTestFiles: false,
+		AutoRetry:       true,
+		Mute:            true,
+		Skip:            false,
+	}
+}
+
 func (p PytestPants) Run(result *RunResult, testCases []plan.TestCase, retry bool) error {
 	testPaths := make([]string, len(testCases))
 	for i, tc := range testCases {

internal/runner/rspec.go

@@ -64,6 +64,17 @@ func (r Rspec) GetFiles() ([]string, error) {
 	return files, nil
 }
 
+func (r Rspec) SupportedFeatures() SupportedFeatures {
+	return SupportedFeatures{
+		SplitByFile:     true,
+		SplitByExample:  true,
+		FilterTestFiles: true,
+		AutoRetry:       true,
+		Mute:            true,
+		Skip:            true,
+	}
+}
+
 // Run executes the test command with the given test cases.
 // If retry is true, it will run the command using the retry test command,
 // otherwise it will use the test command.

internal/runner/types.go

@@ -0,0 +1,39 @@
+package runner
+
+import "github.com/buildkite/test-engine-client/internal/plan"
+
+type RunnerConfig struct {
+	TestRunner             string
+	TestCommand            string
+	TestFilePattern        string
+	TestFileExcludePattern string
+	RetryTestCommand       string
+	TagFilters             string
+	// ResultPath is used internally so bktec can read result from Test Runner.
+	// User typically don't need to worry about setting this except in in RSpec and playwright.
+	// In playwright, for example, it can only be configured via a config file, therefore it's mandatory for user to set.
+	ResultPath string
+}
+
+type TestRunner interface {
+	// Run takes testCases as input, executes the test against the test cases, and mutates the runner.RunResult with the test results.
+	Run(result *RunResult, testCases []plan.TestCase, retry bool) error
+	// GetExamples discovers all tests within given files.
+	// This function is only used for split by example use case. Currently only supported by RSpec.
+	GetExamples(files []string) ([]plan.TestCase, error)
+	// GetFiles discover all test files that the runner should execute.
+	// This is sent to server-side when creating test plan.
+	// This is also used to obtain a fallback non-intelligent test splitting mechanism.
+	GetFiles() ([]string, error)
+	Name() string
+	SupportedFeatures() SupportedFeatures
+}
+
+type SupportedFeatures struct {
+	SplitByFile     bool
+	SplitByExample  bool
+	FilterTestFiles bool
+	AutoRetry       bool
+	Mute            bool
+	Skip            bool
+}