feat: update test harness with metadata assertions #1467

pom.xml

@@ -143,6 +143,13 @@
             <scope>test</scope>
         </dependency>
 
+        <dependency>
+            <groupId>io.cucumber</groupId>
+            <artifactId>cucumber-picocontainer</artifactId>
+            <version>${io.cucumber.version}</version>
+            <scope>test</scope>
+        </dependency>
+
         <dependency>
             <groupId>org.simplify4u</groupId>
             <artifactId>slf4j2-mock</artifactId>
@@ -681,21 +688,6 @@
                                     </arguments>
                                 </configuration>
                             </execution>
-                            <execution>
-                                <id>copy-evaluation-gherkin-tests</id>
-                                <phase>validate</phase>
-                                <goals>
-                                    <goal>exec</goal>
-                                </goals>
-                                <configuration>
-                                    <!-- copy the feature spec we want to test into resources so them can be easily loaded -->
-                                    <executable>cp</executable>
-                                    <arguments>
-                                        <argument>spec/specification/assets/gherkin/evaluation.feature</argument>
-                                        <argument>src/test/resources/features/</argument>
-                                    </arguments>
-                                </configuration>
-                            </execution>
                         </executions>
                     </plugin>
                 </plugins>

src/main/java/dev/openfeature/sdk/ImmutableMetadata.java

@@ -97,6 +97,10 @@ public <T> T getValue(final String key, final Class<T> type) {
         }
     }
 
+    public boolean isEmpty() {
+        return metadata.isEmpty();
+    }
+
     /**
      * Obtain a builder for {@link ImmutableMetadata}.
      */

src/main/java/dev/openfeature/sdk/OpenFeatureClient.java

@@ -217,7 +217,7 @@ private <T> FlagEvaluationDetails<T> evaluateFlag(
             }
         } catch (Exception e) {
             if (details == null) {
-                details = FlagEvaluationDetails.<T>builder().build();
+                details = FlagEvaluationDetails.<T>builder().flagKey(key).build();
             }
             if (e instanceof OpenFeatureError) {
                 details.setErrorCode(((OpenFeatureError) e).getErrorCode());

src/main/java/dev/openfeature/sdk/providers/memory/Flag.java

@@ -1,5 +1,6 @@
 package dev.openfeature.sdk.providers.memory;
 
+import dev.openfeature.sdk.ImmutableMetadata;
 import java.util.Map;
 import lombok.Builder;
 import lombok.Getter;
@@ -18,4 +19,5 @@ public class Flag<T> {
 
     private String defaultVariant;
     private ContextEvaluator<T> contextEvaluator;
+    private ImmutableMetadata flagMetadata;
 }

src/main/java/dev/openfeature/sdk/providers/memory/InMemoryProvider.java

@@ -152,6 +152,7 @@ private <T> ProviderEvaluation<T> getEvaluation(
                 .value(value)
                 .variant(flag.getDefaultVariant())
                 .reason(Reason.STATIC.toString())
+                .flagMetadata(flag.getFlagMetadata())
                 .build();
     }
 }

src/test/java/dev/openfeature/sdk/FlagMetadataTest.java

@@ -1,6 +1,8 @@
 package dev.openfeature.sdk;
 
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
 import org.junit.jupiter.api.DisplayName;
 import org.junit.jupiter.api.Test;
@@ -9,7 +11,7 @@ class FlagMetadataTest {
 
     @Test
     @DisplayName("Test metadata payload construction and retrieval")
-    public void builder_validation() {
+    void builder_validation() {
         // given
         ImmutableMetadata flagMetadata = ImmutableMetadata.builder()
                 .addString("string", "string")
@@ -42,7 +44,7 @@ public void builder_validation() {
 
     @Test
     @DisplayName("Value type mismatch returns a null")
-    public void value_type_validation() {
+    void value_type_validation() {
         // given
         ImmutableMetadata flagMetadata =
                 ImmutableMetadata.builder().addString("string", "string").build();
@@ -53,11 +55,32 @@ public void value_type_validation() {
 
     @Test
     @DisplayName("A null is returned if key does not exist")
-    public void notfound_error_validation() {
+    void notfound_error_validation() {
         // given
         ImmutableMetadata flagMetadata = ImmutableMetadata.builder().build();
 
         // then
         assertThat(flagMetadata.getBoolean("string")).isNull();
     }
+
+    @Test
+    @DisplayName("isEmpty returns true iff the metadata is empty")
+    void isEmpty_returns_true_if_metadata_is_empty() {
+        // given
+        ImmutableMetadata flagMetadata = ImmutableMetadata.builder().build();
+
+        // then
+        assertTrue(flagMetadata.isEmpty());
+    }
+
+    @Test
+    @DisplayName("isEmpty returns false iff the metadata is not empty")
+    void isEmpty_returns_false_if_metadata_is_not_empty() {
+        // given
+        ImmutableMetadata flagMetadata =
+                ImmutableMetadata.builder().addString("a", "b").build();
+
+        // then
+        assertFalse(flagMetadata.isEmpty());
+    }
 }

src/test/java/dev/openfeature/sdk/e2e/EvaluationTest.java

@@ -1,16 +1,18 @@
 package dev.openfeature.sdk.e2e;
 
 import static io.cucumber.junit.platform.engine.Constants.GLUE_PROPERTY_NAME;
+import static io.cucumber.junit.platform.engine.Constants.OBJECT_FACTORY_PROPERTY_NAME;
 import static io.cucumber.junit.platform.engine.Constants.PLUGIN_PROPERTY_NAME;
 
 import org.junit.platform.suite.api.ConfigurationParameter;
 import org.junit.platform.suite.api.IncludeEngines;
-import org.junit.platform.suite.api.SelectClasspathResource;
+import org.junit.platform.suite.api.SelectDirectories;
 import org.junit.platform.suite.api.Suite;
 
 @Suite
 @IncludeEngines("cucumber")
-@SelectClasspathResource("features/evaluation.feature")
+@SelectDirectories("spec/specification/assets/gherkin")
 @ConfigurationParameter(key = PLUGIN_PROPERTY_NAME, value = "pretty")
-@ConfigurationParameter(key = GLUE_PROPERTY_NAME, value = "dev.openfeature.sdk.e2e.evaluation")
+@ConfigurationParameter(key = GLUE_PROPERTY_NAME, value = "dev.openfeature.sdk.e2e.steps")
+@ConfigurationParameter(key = OBJECT_FACTORY_PROPERTY_NAME, value = "io.cucumber.picocontainer.PicoFactory")
 public class EvaluationTest {}

src/test/java/dev/openfeature/sdk/e2e/Flag.java

@@ -0,0 +1,13 @@
+package dev.openfeature.sdk.e2e;
+
+public class Flag {
+    public String name;
+    public Object defaultValue;
+    public String type;
+
+    public Flag(String type, String name, Object defaultValue) {
+        this.name = name;
+        this.defaultValue = defaultValue;
+        this.type = type;
+    }
+}

src/test/java/dev/openfeature/sdk/e2e/MockHook.java

@@ -0,0 +1,50 @@
+package dev.openfeature.sdk.e2e;
+
+import dev.openfeature.sdk.EvaluationContext;
+import dev.openfeature.sdk.FlagEvaluationDetails;
+import dev.openfeature.sdk.Hook;
+import dev.openfeature.sdk.HookContext;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+import lombok.Getter;
+
+public class MockHook implements Hook {
+    @Getter
+    private boolean beforeCalled;
+
+    @Getter
+    private boolean afterCalled;
+
+    @Getter
+    private boolean errorCalled;
+
+    @Getter
+    private boolean finallyAfterCalled;
+
+    @Getter
+    private final Map<String, FlagEvaluationDetails> evaluationDetails = new HashMap<>();
+
+    @Override
+    public Optional<EvaluationContext> before(HookContext ctx, Map hints) {
+        beforeCalled = true;
+        return Optional.of(ctx.getCtx());
+    }
+
+    @Override
+    public void after(HookContext ctx, FlagEvaluationDetails details, Map hints) {
+        afterCalled = true;
+        evaluationDetails.put("after", details);
+    }
+
+    @Override
+    public void error(HookContext ctx, Exception error, Map hints) {
+        errorCalled = true;
+    }
+
+    @Override
+    public void finallyAfter(HookContext ctx, FlagEvaluationDetails details, Map hints) {
+        finallyAfterCalled = true;
+        evaluationDetails.put("finally", details);
+    }
+}

src/test/java/dev/openfeature/sdk/e2e/State.java

@@ -0,0 +1,13 @@
+package dev.openfeature.sdk.e2e;
+
+import dev.openfeature.sdk.Client;
+import dev.openfeature.sdk.FlagEvaluationDetails;
+import dev.openfeature.sdk.MutableContext;
+
+public class State {
+    public Client client;
+    public Flag flag;
+    public MutableContext context = new MutableContext();
+    public FlagEvaluationDetails evaluation;
+    public MockHook hook;
+}

src/test/java/dev/openfeature/sdk/e2e/Utils.java

@@ -0,0 +1,28 @@
+package dev.openfeature.sdk.e2e;
+
+import java.util.Objects;
+
+public final class Utils {
+
+    private Utils() {}
+
+    public static Object convert(String value, String type) {
+        if (Objects.equals(value, "null")) {
+            return null;
+        }
+        switch (type.toLowerCase()) {
+            case "boolean":
+                return Boolean.parseBoolean(value);
+            case "string":
+                return value;
+            case "integer":
+                return Integer.parseInt(value);
+            case "float":
+            case "double":
+                return Double.parseDouble(value);
+            case "long":
+                return Long.parseLong(value);
+        }
+        throw new RuntimeException("Unknown config type: " + type);
+    }
+}

src/test/java/dev/openfeature/sdk/e2e/steps/FlagStepDefinitions.java

@@ -0,0 +1,104 @@
+package dev.openfeature.sdk.e2e.steps;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import dev.openfeature.sdk.FlagEvaluationDetails;
+import dev.openfeature.sdk.ImmutableMetadata;
+import dev.openfeature.sdk.Value;
+import dev.openfeature.sdk.e2e.Flag;
+import dev.openfeature.sdk.e2e.State;
+import dev.openfeature.sdk.e2e.Utils;
+import io.cucumber.datatable.DataTable;
+import io.cucumber.java.en.Given;
+import io.cucumber.java.en.Then;
+import io.cucumber.java.en.When;
+import java.lang.reflect.Field;
+import java.util.HashMap;
+import java.util.List;
+
+public class FlagStepDefinitions {
+    private final State state;
+
+    public FlagStepDefinitions(State state) {
+        this.state = state;
+    }
+
+    @Given("a {}-flag with key {string} and a default value {string}")
+    public void givenAFlag(String type, String name, String defaultValue) {
+        state.flag = new Flag(type, name, Utils.convert(defaultValue, type));
+    }
+
+    @When("the flag was evaluated with details")
+    public void the_flag_was_evaluated_with_details() {
+        FlagEvaluationDetails details;
+        switch (state.flag.type.toLowerCase()) {
+            case "string":
+                details =
+                        state.client.getStringDetails(state.flag.name, (String) state.flag.defaultValue, state.context);
+                break;
+            case "boolean":
+                details = state.client.getBooleanDetails(
+                        state.flag.name, (Boolean) state.flag.defaultValue, state.context);
+                break;
+            case "float":
+                details =
+                        state.client.getDoubleDetails(state.flag.name, (Double) state.flag.defaultValue, state.context);
+                break;
+            case "integer":
+                details = state.client.getIntegerDetails(
+                        state.flag.name, (Integer) state.flag.defaultValue, state.context);
+                break;
+            case "object":
+                details =
+                        state.client.getObjectDetails(state.flag.name, (Value) state.flag.defaultValue, state.context);
+                break;
+            default:
+                throw new AssertionError();
+        }
+        state.evaluation = details;
+    }
+
+    @Then("the resolved details value should be {string}")
+    public void the_resolved_details_value_should_be(String value) {
+        assertThat(state.evaluation.getValue()).isEqualTo(Utils.convert(value, state.flag.type));
+    }
+
+    @Then("the reason should be {string}")
+    public void the_reason_should_be(String reason) {
+        assertThat(state.evaluation.getReason()).isEqualTo(reason);
+    }
+
+    @Then("the variant should be {string}")
+    public void the_variant_should_be(String variant) {
+        assertThat(state.evaluation.getVariant()).isEqualTo(variant);
+    }
+
+    @Then("the resolved metadata value \"{}\" with type \"{}\" should be \"{}\"")
+    public void theResolvedMetadataValueShouldBe(String key, String type, String value)
+            throws NoSuchFieldException, IllegalAccessException {
+        Field f = state.evaluation.getFlagMetadata().getClass().getDeclaredField("metadata");
+        f.setAccessible(true);
+        HashMap<String, Object> metadata = (HashMap<String, Object>) f.get(state.evaluation.getFlagMetadata());
+        assertThat(metadata).containsEntry(key, Utils.convert(value, type));
+    }
+
+    @Then("the resolved metadata is empty")
+    public void theResolvedMetadataIsEmpty() {
+        assertThat(state.evaluation.getFlagMetadata().isEmpty()).isTrue();
+    }
+
+    @Then("the resolved metadata should contain")
+    public void theResolvedMetadataShouldContain(DataTable dataTable) {
+        ImmutableMetadata evaluationMetadata = state.evaluation.getFlagMetadata();
+        List<List<String>> asLists = dataTable.asLists();
+        for (int i = 1; i < asLists.size(); i++) { // skip the header of the table
+            List<String> line = asLists.get(i);
+            String key = line.get(0);
+            String metadataType = line.get(1);
+            Object value = Utils.convert(line.get(2), metadataType);
+
+            assertThat(value).isNotNull();
+            assertThat(evaluationMetadata.getValue(key, value.getClass())).isEqualTo(value);
+        }
+    }
+}