test: add comprehensive Cucumber BDD test suite for Superposition API

flake.nix

@@ -83,11 +83,14 @@
               uv
               # go client
               # go
+              playwright-driver.browsers
             ];
 
             shellHook = ''
                 # If it exists from the host system, kill it
                 unset DEVELOPER_DIR
+                export PLAYWRIGHT_BROWSERS_PATH=${pkgs.playwright-driver.browsers}
+                export PLAYWRIGHT_SKIP_VALIDATE_HOST_REQUIREMENTS=true
             '';
           };
 

makefile

@@ -246,6 +246,16 @@ test: setup frontend superposition
 				'http://localhost:8080/health' 2>&1 > /dev/null
 	cd tests && bun test:clean
 	$(MAKE) bindings-test
+	@echo "Installing cucumber test dependencies"
+	cd tests/cucumber && npm ci
+	@echo "Running API tests via cucumber"
+	cd tests/cucumber && npm run test:api
+	@echo "Running UI tests via cucumber"
+	@if [ -z "$$PLAYWRIGHT_BROWSERS_PATH" ]; then \
+		echo "Installing Playwright browsers (not in nix shell)..."; \
+		cd tests/cucumber && npx playwright install --with-deps chromium; \
+	fi
+	cd tests/cucumber && npm run test:ui
 	$(MAKE) kill
 
 ## npm run test

tests/cucumber/.gitignore

@@ -0,0 +1,4 @@
+node_modules/
+dist/
+reports/
+*.lock

tests/cucumber/cucumber.js

@@ -0,0 +1,23 @@
+/**
+ * Cucumber profiles for API and UI testing.
+ *
+ * Both profiles run the SAME Gherkin feature files from features/.
+ * The only difference is which step definitions and support files are loaded:
+ *
+ *   npm run test:api  → step_definitions/ + support/    (drives the SDK)
+ *   npm run test:ui   → step_definitions_ui/ + support_ui/ (drives Playwright)
+ */
+export default function () {
+  return {
+    api: {
+      import: ["step_definitions/**/*.ts", "support/**/*.ts"],
+      format: ["progress", "html:reports/api-report.html"],
+      formatOptions: { snippetInterface: "async-await" },
+    },
+    ui: {
+      import: ["step_definitions_ui/**/*.ts", "support_ui/**/*.ts"],
+      format: ["progress", "html:reports/ui-report.html"],
+      formatOptions: { snippetInterface: "async-await" },
+    },
+  };
+}

tests/cucumber/features/config.feature

@@ -0,0 +1,30 @@
+@api @config @config_retrieval
+Feature: Configuration Retrieval and Versioning
+  As a developer
+  I want to retrieve configurations and manage versions
+  So that I can serve the right config values to my application
+
+  Background:
+    Given an organisation and workspace exist
+    And a test default config exists for config retrieval
+
+  # ── GetConfig ──────────────────────────────────────────────────────
+
+  Scenario: Get configuration with context
+    When I get the config with the test config key prefix
+    Then the operation should succeed
+    And the response should have a version
+
+  Scenario: Pin workspace to a config version and verify
+    Given I know the current config version
+    When I pin the workspace to that config version
+    And I get the config again
+    Then the config version should match the pinned version
+    When I unpin the workspace config version
+    Then the workspace config version should be unset
+
+  # ── ListVersions ───────────────────────────────────────────────────
+
+  Scenario: List configuration versions
+    When I list config versions with count 10 and page 1
+    Then the operation should succeed

tests/cucumber/features/context.feature

@@ -0,0 +1,65 @@
+@api @context
+Feature: Context Management
+  As a configuration administrator
+  I want to manage contexts with condition-based overrides
+  So that configurations vary based on runtime conditions
+
+  Background:
+    Given an organisation and workspace exist
+    And dimensions and default configs are set up for context tests
+
+  # ── Create ─────────────────────────────────────────────────────────
+
+  Scenario: Create a context with overrides
+    When I create a context with condition "os" equals "android" and override "ctx-config-key" to "android-value"
+    Then the operation should succeed
+    And the response should have a context ID
+
+  # ── Get ────────────────────────────────────────────────────────────
+
+  Scenario: Get a context by ID
+    Given a context exists with condition "os" equals "ios" and override "ctx-config-key" to "ios-value"
+    When I get the context by its ID
+    Then the response should include the override for "ctx-config-key"
+
+  # ── List ───────────────────────────────────────────────────────────
+
+  Scenario: List all contexts
+    Given a context exists with condition "os" equals "web" and override "ctx-config-key" to "web-value"
+    When I list all contexts
+    Then the response should contain a list
+    And the list should contain the created context
+
+  # ── Update Override ────────────────────────────────────────────────
+
+  Scenario: Update a context override
+    Given a context exists with condition "os" equals "android" and override "ctx-config-key" to "old-value"
+    When I update the context override for "ctx-config-key" to "new-value"
+    Then the operation should succeed
+
+  # ── Move Context ───────────────────────────────────────────────────
+
+  Scenario: Move a context to a different condition
+    Given a context exists with condition "os" equals "android" and override "ctx-config-key" to "move-value"
+    When I move the context to condition "os" equals "ios"
+    Then the operation should succeed
+
+  # ── Delete ─────────────────────────────────────────────────────────
+
+  Scenario: Delete a context
+    Given a context exists with condition "os" equals "android" and override "ctx-config-key" to "delete-value"
+    When I delete the context
+    Then the operation should succeed
+
+  # ── Bulk Operations ────────────────────────────────────────────────
+
+  Scenario: Perform bulk context operations
+    When I perform a bulk operation to create contexts for "os" values "android,ios"
+    Then the operation should succeed
+
+  # ── Weight Recompute ───────────────────────────────────────────────
+
+  Scenario: Recompute context weights
+    Given contexts exist for weight recompute
+    When I trigger weight recomputation
+    Then the operation should succeed

tests/cucumber/features/default_config.feature

@@ -0,0 +1,74 @@
+@api @config @default_config
+Feature: Default Configuration Management
+  As a developer
+  I want to manage default configurations with schema validation
+  So that my application has well-defined configuration defaults
+
+  Background:
+    Given an organisation and workspace exist
+    And validation functions are set up
+
+  # ── Create ─────────────────────────────────────────────────────────
+
+  Scenario: Create a valid default config
+    When I create a default config with key "test-key" and value:
+      | name      | Test User |
+      | age       | 30        |
+    And the schema requires "name" as string and "age" as number with minimum 0
+    Then the operation should succeed
+
+  Scenario: Fail to create config with invalid schema type
+    When I create a default config with key "test-key-2" and an invalid schema type "invalid-type"
+    Then the operation should fail with error matching "Invalid JSON schema"
+
+  Scenario: Fail to create config with empty schema
+    When I create a default config with key "test-key-2" and an empty schema
+    Then the operation should fail with error matching "Schema cannot be empty"
+
+  Scenario: Fail to create config when value violates schema constraints
+    When I create a default config with key "test-key-2" where age is -5 but minimum is 0
+    Then the operation should fail with error matching "value is too small, minimum is 0"
+
+  Scenario: Fail to create config when function validation fails
+    When I create a default config with key "test-key-2" using validation function "false_validation"
+    Then the operation should fail with error matching "validation failed"
+
+  Scenario: Create config with value_compute function
+    When I create a default config with key "test-key-3" using compute function "auto_fn"
+    Then the operation should succeed
+    And the response should have value_compute_function_name "auto_fn"
+
+  Scenario: Fail to create config with non-existent function
+    When I create a default config with key "test-key-2" using validation function "non_existent_function"
+    Then the operation should fail with error matching "published code does not exist"
+
+  # ── Update ─────────────────────────────────────────────────────────
+
+  Scenario: Update an existing default config value
+    Given a default config exists with key "test-key-upd" and value "Test User" age 30
+    When I update the default config "test-key-upd" with value "Updated User" age 35
+    Then the response value should have name "Updated User" and age 35
+
+  Scenario: Update schema and value together
+    Given a default config exists with key "test-key-upd2" and value "Test User" age 30
+    When I update default config "test-key-upd2" schema to add email field and set value with email "updated@example.com"
+    Then the response value should include email "updated@example.com"
+
+  Scenario: Fail to update a non-existent key
+    When I update default config "non_existent_key" with a new value
+    Then the operation should fail with error matching "No record found"
+
+  Scenario: Fail to update with invalid schema
+    Given a default config exists with key "test-key-upd3" and value "Test User" age 30
+    When I update default config "test-key-upd3" schema to an invalid type
+    Then the operation should fail with error matching "Invalid JSON schema"
+
+  Scenario: Fail to update when value misses required field
+    Given a default config exists with key "test-key-upd4" and requires name and email
+    When I update default config "test-key-upd4" value without the required email field
+    Then the operation should fail with error matching "required property"
+
+  Scenario: Update config with a validation function
+    Given a default config exists with key "test-key-upd5" and value "Test User" age 30
+    When I update default config "test-key-upd5" validation function to "true_function"
+    Then the response should have value_validation_function_name "true_function"

tests/cucumber/features/dimension.feature

@@ -0,0 +1,58 @@
+@api @dimension
+Feature: Dimension Management
+  As a configuration administrator
+  I want to manage dimensions (context keys)
+  So that I can define the axes along which configuration varies
+
+  Background:
+    Given an organisation and workspace exist
+
+  # ── Create ─────────────────────────────────────────────────────────
+
+  Scenario: Create a string dimension
+    When I create a dimension with name "test-dim-str" and schema type "string"
+    Then the operation should succeed
+    And the response should have dimension name "test-dim-str"
+
+  Scenario: Create an enum dimension
+    When I create a dimension with name "test-dim-enum" and enum values "small,big,otherwise"
+    Then the operation should succeed
+
+  Scenario: Create a boolean dimension
+    When I create a dimension with name "test-dim-bool" and schema type "boolean"
+    Then the operation should succeed
+
+  # ── Get ────────────────────────────────────────────────────────────
+
+  Scenario: Get a dimension by name
+    Given a dimension "test-dim-get" exists with schema type "string"
+    When I get dimension "test-dim-get"
+    Then the response should have dimension name "test-dim-get"
+
+  # ── List ───────────────────────────────────────────────────────────
+
+  Scenario: List all dimensions
+    Given a dimension "test-dim-list" exists with schema type "string"
+    When I list all dimensions
+    Then the response should contain a list
+    And the list should contain dimension "test-dim-list"
+
+  # ── Update ─────────────────────────────────────────────────────────
+
+  Scenario: Update a dimension's description
+    Given a dimension "test-dim-upd" exists with schema type "string"
+    When I update dimension "test-dim-upd" description to "Updated description"
+    Then the operation should succeed
+
+  # ── Delete ─────────────────────────────────────────────────────────
+
+  Scenario: Delete a dimension
+    Given a dimension "test-dim-del" exists with schema type "string"
+    When I delete dimension "test-dim-del"
+    Then the operation should succeed
+
+  # ── Error Cases ────────────────────────────────────────────────────
+
+  Scenario: Fail to create a dimension with invalid schema
+    When I create a dimension with name "test-dim-invalid" and schema type "invalid-type"
+    Then the operation should fail

tests/cucumber/features/experiment.feature

@@ -0,0 +1,69 @@
+@api @experiment
+Feature: Experiment Management
+  As a product manager
+  I want to manage A/B test experiments
+  So that I can test configuration variants with controlled traffic
+
+  Background:
+    Given an organisation and workspace exist
+    And dimensions and default configs are set up for experiment tests
+
+  # ── Create ─────────────────────────────────────────────────────────
+
+  Scenario: Create an experiment with control and experimental variants
+    When I create an experiment with name "exp-test" and context "os" equals "android"
+    And the experiment has a control variant with override "exp-config-key" = "control-val"
+    And the experiment has an experimental variant with override "exp-config-key" = "experimental-val"
+    Then the operation should succeed
+    And the response should have experiment status "CREATED"
+    And the response should have 2 variants
+
+  # ── Get ────────────────────────────────────────────────────────────
+
+  Scenario: Get an experiment by ID
+    Given an experiment "exp-get" exists with context "os" equals "ios"
+    When I get the experiment by its ID
+    Then the response should have the experiment name
+    And the response should have experiment status "CREATED"
+
+  # ── List ───────────────────────────────────────────────────────────
+
+  Scenario: List experiments
+    Given an experiment "exp-list" exists with context "os" equals "android"
+    When I list experiments
+    Then the response should contain a list
+    And the list should contain the created experiment
+
+  # ── Ramp ───────────────────────────────────────────────────────────
+
+  Scenario: Ramp an experiment to 50% traffic
+    Given an experiment "exp-ramp" exists with context "os" equals "android"
+    When I ramp the experiment to 50 percent traffic
+    Then the operation should succeed
+    And the experiment status should be "INPROGRESS"
+
+  # ── Update Overrides ───────────────────────────────────────────────
+
+  Scenario: Update experiment variant overrides
+    Given an experiment "exp-override" exists with context "os" equals "android"
+    When I update the experimental variant override for "exp-config-key" to "updated-val"
+    Then the operation should succeed
+
+  # ── Conclude ───────────────────────────────────────────────────────
+
+  Scenario: Conclude an experiment
+    Given an experiment "exp-conclude" exists and is ramped to 50 percent
+    When I conclude the experiment with the experimental variant
+    Then the experiment status should be "CONCLUDED"
+
+  # ── Discard ────────────────────────────────────────────────────────
+
+  Scenario: Discard a created experiment
+    Given an experiment "exp-discard" exists with context "os" equals "ios"
+    When I discard the experiment
+    Then the experiment status should be "DISCARDED"
+
+  Scenario: Discard an in-progress experiment
+    Given an experiment "exp-discard-prog" exists and is ramped to 50 percent
+    When I discard the experiment
+    Then the experiment status should be "DISCARDED"