[SDTEST-2812] Sanity checks for Ruby test optimization setup

README.md

@@ -8,6 +8,10 @@ Currently supported languages and frameworks:
 
 - Ruby (RSpec, Minitest)
 
+DDTest requires that your project is correctly set up for Datadog Test Optimization with the native library for your language. Minimum supported library versions:
+
+- Ruby: `datadog-ci` gem **1.23.0** or higher
+
 ## Installation
 
 ### From Source
@@ -112,15 +116,15 @@ In CI‑node mode, DDTest also fans out across local CPUs on that node and furth
 
 ### Settings (flags and environment variables)
 
-| CLI flag            | Environment variable                          |    Default | What it does                                                                                                                                                 |
-| ------------------- | --------------------------------------------- | ---------: | ------------------------------------------------------------------------------------------------------------------------------------------------------------ |
-| `--platform`        | `DD_TEST_OPTIMIZATION_RUNNER_PLATFORM`        |     `ruby` | Language/platform (currently supported values: `ruby`).                                                                                                      |
-| `--framework`       | `DD_TEST_OPTIMIZATION_RUNNER_FRAMEWORK`       |    `rspec` | Test framework (currently supported values: `rspec`, `minitest`).                                                                                            |
-| `--min-parallelism` | `DD_TEST_OPTIMIZATION_RUNNER_MIN_PARALLELISM` | vCPU count | Minimum workers to use for the split.                                                                                                                        |
-| `--max-parallelism` | `DD_TEST_OPTIMIZATION_RUNNER_MAX_PARALLELISM` | vCPU count | Maximum workers to use for the split.                                                                                                                        |
-| `--ci-node`         | `DD_TEST_OPTIMIZATION_RUNNER_CI_NODE`         | `-1` (off) | Restrict this run to the slice assigned to node **N** (0‑indexed). Also parallelizes within the node across its CPUs.                                        |
-| `--worker-env`      | `DD_TEST_OPTIMIZATION_RUNNER_WORKER_ENV`      |       `""` | Template env vars per local worker (e.g., isolate DBs): `--worker-env "DATABASE_NAME_TEST=app_test{{nodeIndex}}"`.                                           |
-| `--command`         | `DD_TEST_OPTIMIZATION_RUNNER_COMMAND`         |       `""` | Override the default test command used by the framework. When provided, takes precedence over auto-detection (e.g., `--command "bundle exec custom-rspec"`). |
+| CLI flag            | Environment variable                          |    Default | What it does                                                                                                                                                                   |
+| ------------------- | --------------------------------------------- | ---------: | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
+| `--platform`        | `DD_TEST_OPTIMIZATION_RUNNER_PLATFORM`        |     `ruby` | Language/platform (currently supported values: `ruby`).                                                                                                                        |
+| `--framework`       | `DD_TEST_OPTIMIZATION_RUNNER_FRAMEWORK`       |    `rspec` | Test framework (currently supported values: `rspec`, `minitest`).                                                                                                              |
+| `--min-parallelism` | `DD_TEST_OPTIMIZATION_RUNNER_MIN_PARALLELISM` | vCPU count | Minimum workers to use for the split.                                                                                                                                          |
+| `--max-parallelism` | `DD_TEST_OPTIMIZATION_RUNNER_MAX_PARALLELISM` | vCPU count | Maximum workers to use for the split.                                                                                                                                          |
+| `--ci-node`         | `DD_TEST_OPTIMIZATION_RUNNER_CI_NODE`         | `-1` (off) | Restrict this run to the slice assigned to node **N** (0‑indexed). Also parallelizes within the node across its CPUs.                                                          |
+| `--worker-env`      | `DD_TEST_OPTIMIZATION_RUNNER_WORKER_ENV`      |       `""` | Template env vars per local worker (e.g., isolate DBs): `--worker-env "DATABASE_NAME_TEST=app_test{{nodeIndex}}"`.                                                             |
+| `--command`         | `DD_TEST_OPTIMIZATION_RUNNER_COMMAND`         |       `""` | Override the default test command used by the framework. When provided, takes precedence over auto-detection (e.g., `--command "bundle exec custom-rspec"`).                   |
 | `--tests-location`  | `DD_TEST_OPTIMIZATION_RUNNER_TESTS_LOCATION`  |       `""` | Custom glob pattern to discover test files (e.g., `--tests-location "custom/spec/**/*_spec.rb"`). Defaults to `spec/**/*_spec.rb` for RSpec, `test/**/*_test.rb` for Minitest. |
 
 #### Note about the `--command` flag

internal/platform/platform.go

@@ -11,6 +11,7 @@ type Platform interface {
 	Name() string
 	CreateTagsMap() (map[string]string, error)
 	DetectFramework() (framework.Framework, error)
+	SanityCheck() error
 }
 
 // PlatformDetector defines interface for detecting platforms - needed to allow mocking in tests
@@ -35,6 +36,10 @@ func DetectPlatform() (Platform, error) {
 		return nil, fmt.Errorf("unsupported platform: %s", platformName)
 	}
 
+	if err := platform.SanityCheck(); err != nil {
+		return nil, fmt.Errorf("sanity check failed for platform %s: %w", platform.Name(), err)
+	}
+
 	return platform, nil
 }
 

internal/platform/ruby.go

@@ -7,16 +7,23 @@ import (
 	"fmt"
 	"maps"
 	"os"
+	"strings"
 
 	"github.com/DataDog/ddtest/internal/constants"
 	"github.com/DataDog/ddtest/internal/ext"
 	"github.com/DataDog/ddtest/internal/framework"
 	"github.com/DataDog/ddtest/internal/settings"
+	"github.com/DataDog/ddtest/internal/version"
 )
 
 //go:embed scripts/ruby_env.rb
 var rubyEnvScript string
 
+const (
+	requiredGemName       = "datadog-ci"
+	requiredGemMinVersion = "1.23.0"
+)
+
 type Ruby struct {
 	executor ext.CommandExecutor
 }
@@ -80,3 +87,70 @@ func (r *Ruby) DetectFramework() (framework.Framework, error) {
 		return nil, fmt.Errorf("framework '%s' is not supported by platform 'ruby'", frameworkName)
 	}
 }
+
+func (r *Ruby) SanityCheck() error {
+	args := []string{"info", requiredGemName}
+	output, err := r.executor.CombinedOutput(context.Background(), "bundle", args, nil)
+	if err != nil {
+		message := strings.TrimSpace(string(output))
+		if message == "" {
+			return fmt.Errorf("bundle info datadog-ci command failed: %w", err)
+		}
+		return fmt.Errorf("bundle info datadog-ci command failed: %s", message)
+	}
+
+	requiredVersion, err := version.Parse(requiredGemMinVersion)
+	if err != nil {
+		return err
+	}
+
+	gemVersion, err := parseBundlerInfoVersion(string(output), requiredGemName)
+	if err != nil {
+		return err
+	}
+
+	if gemVersion.Compare(requiredVersion) < 0 {
+		return fmt.Errorf("datadog-ci gem version %s is lower than required >= %s", gemVersion.String(), requiredVersion.String())
+	}
+
+	return nil
+}
+
+func parseBundlerInfoVersion(output, gemName string) (version.Version, error) {
+	for line := range strings.SplitSeq(output, "\n") {
+		trimmed := strings.TrimSpace(line)
+		if trimmed == "" {
+			continue
+		}
+
+		if !strings.Contains(trimmed, gemName) {
+			continue
+		}
+
+		start := strings.Index(trimmed, "(")
+		end := strings.Index(trimmed, ")")
+		if start == -1 || end == -1 || end <= start+1 {
+			continue
+		}
+
+		versionToken := strings.TrimSpace(trimmed[start+1 : end])
+		if versionToken == "" {
+			continue
+		}
+
+		fields := strings.Fields(versionToken)
+		versionString := fields[0]
+		if !version.IsValid(versionString) {
+			return version.Version{}, fmt.Errorf("unexpected version format in bundle info output: %q", versionToken)
+		}
+
+		parsed, err := version.Parse(versionString)
+		if err != nil {
+			return version.Version{}, fmt.Errorf("failed to parse version from bundle info output: %w", err)
+		}
+
+		return parsed, nil
+	}
+
+	return version.Version{}, fmt.Errorf("unable to find datadog-ci gem version in bundle info output")
+}

internal/platform/ruby_test.go

@@ -14,23 +14,25 @@ import (
 )
 
 type mockCommandExecutor struct {
-	output      []byte
-	err         error
-	onExecution func(name string, args []string)
+	runErr            error
+	combinedOutput    []byte
+	combinedOutputErr error
+	onRun             func(name string, args []string, envMap map[string]string)
+	onCombinedOutput  func(name string, args []string, envMap map[string]string)
 }
 
 func (m *mockCommandExecutor) CombinedOutput(ctx context.Context, name string, args []string, envMap map[string]string) ([]byte, error) {
-	if m.onExecution != nil {
-		m.onExecution(name, args)
+	if m.onCombinedOutput != nil {
+		m.onCombinedOutput(name, args, envMap)
 	}
-	return m.output, m.err
+	return m.combinedOutput, m.combinedOutputErr
 }
 
 func (m *mockCommandExecutor) Run(ctx context.Context, name string, args []string, envMap map[string]string) error {
-	if m.onExecution != nil {
-		m.onExecution(name, args)
+	if m.onRun != nil {
+		m.onRun(name, args, envMap)
 	}
-	return m.err
+	return m.runErr
 }
 
 func TestRuby_Name(t *testing.T) {
@@ -43,6 +45,78 @@ func TestRuby_Name(t *testing.T) {
 	}
 }
 
+func TestRuby_SanityCheck_Passes(t *testing.T) {
+	mockExecutor := &mockCommandExecutor{
+		combinedOutput: []byte("  * datadog-ci (1.23.1 9d54a15)\n"),
+		onCombinedOutput: func(name string, args []string, envMap map[string]string) {
+			if name != "bundle" {
+				t.Fatalf("expected command 'bundle', got %q", name)
+			}
+			if len(args) != 2 || args[0] != "info" || args[1] != "datadog-ci" {
+				t.Fatalf("unexpected args: %v", args)
+			}
+		},
+	}
+
+	ruby := NewRuby()
+	ruby.executor = mockExecutor
+	if err := ruby.SanityCheck(); err != nil {
+		t.Fatalf("SanityCheck() unexpected error: %v", err)
+	}
+}
+
+func TestRuby_SanityCheck_FailsWhenBundleInfoFails(t *testing.T) {
+	mockExecutor := &mockCommandExecutor{
+		combinedOutput:    []byte("Could not find gem 'datadog-ci'."),
+		combinedOutputErr: &exec.ExitError{},
+	}
+
+	ruby := NewRuby()
+	ruby.executor = mockExecutor
+	err := ruby.SanityCheck()
+	if err == nil {
+		t.Fatal("SanityCheck() expected error when bundle info fails")
+	}
+
+	if !strings.Contains(err.Error(), "Could not find gem") {
+		t.Fatalf("unexpected error: %v", err)
+	}
+}
+
+func TestRuby_SanityCheck_FailsWhenVersionTooLow(t *testing.T) {
+	mockExecutor := &mockCommandExecutor{
+		combinedOutput: []byte("  * datadog-ci (1.22.5)\n"),
+	}
+
+	ruby := NewRuby()
+	ruby.executor = mockExecutor
+	err := ruby.SanityCheck()
+	if err == nil {
+		t.Fatal("SanityCheck() expected error for outdated datadog-ci version")
+	}
+
+	if !strings.Contains(err.Error(), "1.22.5") {
+		t.Fatalf("expected error to mention detected version, got: %v", err)
+	}
+}
+
+func TestRuby_SanityCheck_FailsWhenVersionNotFound(t *testing.T) {
+	mockExecutor := &mockCommandExecutor{
+		combinedOutput: []byte("  * datadog-ci\n    Summary: Datadog Test Optimization for your ruby application\n"),
+	}
+
+	ruby := NewRuby()
+	ruby.executor = mockExecutor
+	err := ruby.SanityCheck()
+	if err == nil {
+		t.Fatal("SanityCheck() expected error when version is not found")
+	}
+
+	if !strings.Contains(err.Error(), "unable to find datadog-ci gem version") {
+		t.Fatalf("unexpected error: %v", err)
+	}
+}
+
 func TestRuby_DetectFramework_RSpec(t *testing.T) {
 	viper.Reset()
 	viper.Set("framework", "rspec")
@@ -140,8 +214,7 @@ func TestRuby_CreateTagsMap_Success(t *testing.T) {
 	}
 
 	mockExecutor := &mockCommandExecutor{
-		err: nil,
-		onExecution: func(name string, args []string) {
+		onRun: func(name string, args []string, envMap map[string]string) {
 			// Verify the command is correct
 			if name != "bundle" {
 				t.Errorf("expected command to be 'bundle', got %q", name)
@@ -209,9 +282,8 @@ func TestRuby_CreateTagsMap_CommandFailure(t *testing.T) {
 	}()
 
 	mockExecutor := &mockCommandExecutor{
-		output: []byte("bundle: command not found"),
-		err:    &exec.ExitError{},
-		onExecution: func(name string, args []string) {
+		runErr: &exec.ExitError{},
+		onRun: func(name string, args []string, envMap map[string]string) {
 			// Command fails, don't create any file
 		},
 	}
@@ -243,8 +315,7 @@ func TestRuby_CreateTagsMap_InvalidJSON(t *testing.T) {
 
 	invalidJSON := `{invalid json}`
 	mockExecutor := &mockCommandExecutor{
-		err: nil,
-		onExecution: func(name string, args []string) {
+		onRun: func(name string, args []string, envMap map[string]string) {
 			// Get the temp file path from the last argument
 			if len(args) < 5 {
 				t.Errorf("expected at least 5 args, got %d", len(args))
@@ -305,24 +376,15 @@ func TestDetectPlatform_Ruby(t *testing.T) {
 	viper.Set("platform", "ruby")
 
 	platform, err := DetectPlatform()
-	if err != nil {
-		t.Fatalf("DetectPlatform failed: %v", err)
-	}
-
-	if platform == nil {
-		t.Error("expected platform to be non-nil")
-	}
-
-	if platform.Name() != "ruby" {
-		t.Errorf("expected platform name to be 'ruby', got %q", platform.Name())
+	if err == nil {
+		t.Errorf("expected error for SanityCheck failure, but got platform: %v", platform)
+	} else if platform != nil {
+		t.Errorf("expected nil platform for SanityCheck failure, but got platform: %v", platform)
 	}
 
-	// Verify it's the correct type and has executor
-	rubyPlatform, ok := platform.(*Ruby)
-	if !ok {
-		t.Error("expected platform to be *Ruby")
-	} else if rubyPlatform.executor == nil {
-		t.Error("expected Ruby platform to have executor")
+	expectedErrorPrefix := "sanity check failed for platform ruby: bundle info datadog-ci command failed"
+	if !strings.Contains(err.Error(), expectedErrorPrefix) {
+		t.Errorf("expected error to contain %q, got %q", expectedErrorPrefix, err.Error())
 	}
 }
 

internal/runner/runner_test.go

@@ -41,6 +41,7 @@ type MockPlatform struct {
 	TagsErr      error
 	Framework    framework.Framework
 	FrameworkErr error
+	SanityErr    error
 }
 
 func (m *MockPlatform) Name() string {
@@ -55,6 +56,10 @@ func (m *MockPlatform) DetectFramework() (framework.Framework, error) {
 	return m.Framework, m.FrameworkErr
 }
 
+func (m *MockPlatform) SanityCheck() error {
+	return m.SanityErr
+}
+
 // MockFramework mocks a testing framework
 type MockFramework struct {
 	FrameworkName string