chore: add integration tests (#77)

* chore: add integration tests

Signed-off-by: Todd Baert <toddbaert@gmail.com>

* improve POM spacing

Signed-off-by: Todd Baert <toddbaert@gmail.com>

Signed-off-by: Todd Baert <toddbaert@gmail.com>

.github/workflows/pullrequest.yml

@@ -9,6 +9,12 @@ permissions:
 jobs:
   build:
     runs-on: ubuntu-latest
+    services:
+      flagd:
+        image: ghcr.io/open-feature/flagd-testbed:latest
+        ports:
+          - 8013:8013
+
     steps:
       - name: Check out the code
         uses: actions/checkout@v3
@@ -28,7 +34,7 @@ jobs:
             ${{ runner.os }}-maven-
 
       - name: Build with Maven
-        run: mvn --batch-mode --update-snapshots verify
+        run: mvn --batch-mode --update-snapshots verify -P integration-test
 
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v2

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "test-harness"]
+	path = test-harness
+	url = https://github.com/open-feature/test-harness

README.md

@@ -112,6 +112,12 @@ We hold regular meetings which you can see [here](https://github.com/open-featur
 
 We are also present on the `#openfeature` channel in the [CNCF slack](https://slack.cncf.io/).
 
+## Developing
+
+### Integration tests
+
+The continuous integration runs a set of [gherkin integration tests](https://github.com/open-feature/test-harness/blob/main/features/evaluation.feature) using [`flagd`](https://github.com/open-feature/flagd). These tests do not run with the default maven profile. If you'd like to run them locally, you can start the flagd testbed with `docker run -p 8013:8013 ghcr.io/open-feature/flagd-testbed:latest` and then run `mvn test -P integration-test`.
+
 ## Releasing
 
 See [releasing](./docs/release.md).

pom.xml

@@ -11,6 +11,8 @@
     <maven.compiler.source>1.8</maven.compiler.source>
     <maven.compiler.target>${maven.compiler.source}</maven.compiler.target>
     <junit.jupiter.version>5.8.1</junit.jupiter.version>
+    <!-- exclusion expression for integration tests -->
+    <testExclusions>**/integration/*.java</testExclusions>
   </properties>
 
   <name>OpenFeature Java SDK</name>
@@ -46,7 +48,6 @@
       <scope>provided</scope>
     </dependency>
 
-
     <dependency>
       <!-- used so that lombok can generate suppressions for spotbugs. It needs to find it on the relevant classpath -->
       <groupId>com.github.spotbugs</groupId>
@@ -96,29 +97,81 @@
       <version>${junit.jupiter.version}</version>
       <scope>test</scope>
     </dependency>
+
     <dependency>
       <groupId>org.junit.jupiter</groupId>
       <artifactId>junit-jupiter-api</artifactId>
       <version>${junit.jupiter.version}</version>
       <scope>test</scope>
     </dependency>
+
     <dependency>
       <groupId>org.junit.jupiter</groupId>
       <artifactId>junit-jupiter-params</artifactId>
       <version>${junit.jupiter.version}</version>
       <scope>test</scope>
     </dependency>
+
     <dependency>
       <groupId>org.junit.platform</groupId>
       <artifactId>junit-platform-suite</artifactId>
-      <version>1.8.1</version>
+      <version>1.9.0</version>
       <scope>test</scope>
     </dependency>
+
+    <dependency>
+      <groupId>io.cucumber</groupId>
+      <artifactId>cucumber-java</artifactId>
+      <scope>test</scope>
+    </dependency>
+
+    <dependency>
+        <groupId>io.cucumber</groupId>
+        <artifactId>cucumber-junit-platform-engine</artifactId>
+        <scope>test</scope>
+    </dependency>
+
+    <dependency>
+      <groupId>com.google.guava</groupId>
+      <artifactId>guava</artifactId>
+      <version>31.1-jre</version>
+      <scope>test</scope>
+    </dependency>
+
+    <dependency>
+      <groupId>dev.openfeature.contrib.providers</groupId>
+      <artifactId>flagd</artifactId>
+      <version>0.3.2</version>
+      <scope>test</scope>
+    </dependency>
+
   </dependencies>
 
+  <dependencyManagement>
+    <dependencies>
+
+        <dependency>
+            <groupId>io.cucumber</groupId>
+            <artifactId>cucumber-bom</artifactId>
+            <version>7.5.0</version>
+            <type>pom</type>
+            <scope>import</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.junit</groupId>
+            <artifactId>junit-bom</artifactId>
+            <version>5.9.0</version>
+            <type>pom</type>
+            <scope>import</scope>
+        </dependency>
+
+    </dependencies>
+  </dependencyManagement>
 
   <build>
     <plugins>
+
       <plugin>
         <artifactId>maven-dependency-plugin</artifactId>
         <version>3.3.0</version>
@@ -137,16 +190,21 @@
             <ignoredUnusedDeclaredDependency>org.junit*</ignoredUnusedDeclaredDependency>
           </ignoredUnusedDeclaredDependencies>
           <ignoredDependencies>
+            <ignoredDependency>com.google.guava*</ignoredDependency>
+            <ignoredDependency>io.cucumber*</ignoredDependency>
+            <ignoredDependency>org.junit*</ignoredDependency>
             <ignoredDependency>com.google.code.findbugs*</ignoredDependency>
             <ignoredDependency>com.github.spotbugs*</ignoredDependency>
             <ignoredDependency>uk.org.lidalia:lidalia-slf4j-ext:*</ignoredDependency>
           </ignoredDependencies>
         </configuration>
       </plugin>
+
       <plugin>
         <artifactId>maven-compiler-plugin</artifactId>
         <version>3.8.1</version>
       </plugin>
+
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-surefire-plugin</artifactId>
@@ -155,6 +213,10 @@
           <argLine>
             ${surefireArgLine}
           </argLine>
+          <excludes>
+            <!-- tests to exclude -->
+            <exclude>${testExclusions}</exclude>
+         </excludes>
         </configuration>
       </plugin>
 
@@ -224,7 +286,7 @@
                 </rule>
               </rules>
             </configuration>
- 	  </execution>
+ 	        </execution>
 
         </executions>
       </plugin>
@@ -257,6 +319,7 @@
           </execution>
         </executions>
       </plugin>
+      
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-javadoc-plugin</artifactId>
@@ -372,6 +435,61 @@
     </plugins>
   </build>
 
+  <profiles>
+    <profile>
+      <!-- this profile handles running the flagd integration tests -->
+      <id>integration-test</id>
+      <properties>
+        <!-- run the integration tests by clearing the exclusions -->
+        <testExclusions></testExclusions>
+      </properties>  
+      <build>
+        <plugins>
+          <!-- pull the gherkin tests as a git submodule  -->
+          <plugin>
+            <groupId>org.codehaus.mojo</groupId>
+            <artifactId>exec-maven-plugin</artifactId>
+            <version>3.1.0</version>
+            <executions>
+              <execution>
+                <id>update-test-harness-submodule</id>
+                <phase>validate</phase>
+                <goals>
+                  <goal>exec</goal>
+                </goals>
+                <configuration>
+                  <!-- run: git submodule update \-\-init \-\-recursive -->
+                  <executable>git</executable>
+                  <arguments>
+                    <argument>submodule</argument>
+                    <argument>update</argument>
+                    <argument>--init</argument>
+                    <argument>--recursive</argument>
+                  </arguments>
+                </configuration>
+              </execution>
+              <execution>
+                <id>copy-gherkin-tests</id>
+                <phase>validate</phase>
+                <goals>
+                  <goal>exec</goal>
+                </goals>
+                <configuration>
+                  <!-- run: cp test-harness/features/evaluation.feature src/test/resources/features/ -->
+                  <executable>cp</executable>
+                  <arguments>
+                    <argument>test-harness/features/evaluation.feature</argument>
+                    <argument>src/test/resources/features/</argument>
+                  </arguments>
+                </configuration>
+              </execution>
+            </executions>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
+  </profiles>
+
   <distributionManagement>
     <snapshotRepository>
       <id>ossrh</id>

src/test/java/dev/openfeature/javasdk/integration/RunCucumberTest.java

@@ -0,0 +1,16 @@
+package dev.openfeature.javasdk.integration;
+
+import org.junit.platform.suite.api.ConfigurationParameter;
+import org.junit.platform.suite.api.IncludeEngines;
+import org.junit.platform.suite.api.SelectClasspathResource;
+import org.junit.platform.suite.api.Suite;
+
+import static io.cucumber.junit.platform.engine.Constants.PLUGIN_PROPERTY_NAME;
+
+@Suite
+@IncludeEngines("cucumber")
+@SelectClasspathResource("features")
+@ConfigurationParameter(key = PLUGIN_PROPERTY_NAME, value = "pretty")
+public class RunCucumberTest {
+  
+}

src/test/java/dev/openfeature/javasdk/integration/StepDefinitions.java

@@ -0,0 +1,282 @@
+package dev.openfeature.javasdk.integration;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assumptions.assumeFalse;
+
+import dev.openfeature.contrib.providers.flagd.FlagdProvider;
+import dev.openfeature.javasdk.Client;
+import dev.openfeature.javasdk.ErrorCode;
+import dev.openfeature.javasdk.EvaluationContext;
+import dev.openfeature.javasdk.FlagEvaluationDetails;
+import dev.openfeature.javasdk.OpenFeatureAPI;
+import dev.openfeature.javasdk.Reason;
+import dev.openfeature.javasdk.Structure;
+import dev.openfeature.javasdk.Value;
+import io.cucumber.java.BeforeAll;
+import io.cucumber.java.en.Then;
+import io.cucumber.java.en.When;
+
+public class StepDefinitions {
+
+    private static Client client;
+    private boolean booleanFlagValue;
+    private String stringFlagValue;
+    private int intFlagValue;
+    private double doubleFlagValue;
+    private Value objectFlagValue;
+
+    private FlagEvaluationDetails<Boolean> booleanFlagDetails;
+    private FlagEvaluationDetails<String> stringFlagDetails;
+    private FlagEvaluationDetails<Integer> intFlagDetails;
+    private FlagEvaluationDetails<Double> doubleFlagDetails;
+    private FlagEvaluationDetails<Value> objectFlagDetails;
+
+    private String contextAwareFlagKey;
+    private String contextAwareDefaultValue;
+    private EvaluationContext context;
+    private String contextAwareValue;
+
+    private String notFoundFlagKey;
+    private String notFoundDefaultValue;
+    private FlagEvaluationDetails<String> notFoundDetails;
+    private String typeErrorFlagKey;
+    private int typeErrorDefaultValue;
+    private FlagEvaluationDetails<Integer> typeErrorDetails;
+
+    @BeforeAll()
+    public static void setup() {
+        OpenFeatureAPI.getInstance().setProvider(new FlagdProvider());
+        client = OpenFeatureAPI.getInstance().getClient();
+    }
+
+    /*
+     * Basic evaluation
+     */
+
+    // boolean value
+    @When("a boolean flag with key {string} is evaluated with default value {string}")
+    public void a_boolean_flag_with_key_boolean_flag_is_evaluated_with_default_value_false(String flagKey,
+            String defaultValue) {
+        this.booleanFlagValue = client.getBooleanValue(flagKey, Boolean.valueOf(defaultValue));
+    }
+
+    @Then("the resolved boolean value should be {string}")
+    public void the_resolved_boolean_value_should_be_true(String expected) {
+        assertEquals(Boolean.valueOf(expected), this.booleanFlagValue);
+    }
+
+    // string value
+    @When("a string flag with key {string} is evaluated with default value {string}")
+    public void a_string_flag_with_key_is_evaluated_with_default_value(String flagKey, String defaultValue) {
+        this.stringFlagValue = client.getStringValue(flagKey, defaultValue);
+    }
+
+    @Then("the resolved string value should be {string}")
+    public void the_resolved_string_value_should_be(String expected) {
+        assertEquals(expected, this.stringFlagValue);
+    }
+
+    // integer value
+    @When("an integer flag with key {string} is evaluated with default value {int}")
+    public void an_integer_flag_with_key_is_evaluated_with_default_value(String flagKey, Integer defaultValue) {
+        this.intFlagValue = client.getIntegerValue(flagKey, defaultValue);
+    }
+
+    @Then("the resolved integer value should be {int}")
+    public void the_resolved_integer_value_should_be(int expected) {
+        assertEquals(expected, this.intFlagValue);
+    }
+
+    // float/double value
+    @When("a float flag with key {string} is evaluated with default value {double}")
+    public void a_float_flag_with_key_is_evaluated_with_default_value(String flagKey, double defaultValue) {
+        this.doubleFlagValue = client.getDoubleValue(flagKey, defaultValue);
+    }
+
+    @Then("the resolved float value should be {double}")
+    public void the_resolved_float_value_should_be(double expected) {
+        assertEquals(expected, this.doubleFlagValue);
+    }
+
+    // object value
+    @When("an object flag with key {string} is evaluated with a null default value")
+    public void an_object_flag_with_key_is_evaluated_with_a_null_default_value(String flagKey) {
+        this.objectFlagValue = client.getObjectValue(flagKey, new Value());
+    }
+
+    @Then("the resolved object value should be contain fields {string}, {string}, and {string}, with values {string}, {string} and {int}, respectively")
+    public void the_resolved_object_value_should_be_contain_fields_and_with_values_and_respectively(String boolField,
+            String stringField, String numberField, String boolValue, String stringValue, int numberValue) {
+        Structure structure = this.objectFlagValue.asStructure();
+
+        assertEquals(Boolean.valueOf(boolValue), structure.asMap().get(boolField).asBoolean());
+        assertEquals(stringValue, structure.asMap().get(stringField).asString());
+        assertEquals(numberValue, structure.asMap().get(numberField).asInteger());
+    }
+
+    /*
+     * Detailed evaluation
+     */
+
+    // boolean details
+    @When("a boolean flag with key {string} is evaluated with details and default value {string}")
+    public void a_boolean_flag_with_key_is_evaluated_with_details_and_default_value(String flagKey,
+            String defaultValue) {
+        this.booleanFlagDetails = client.getBooleanDetails(flagKey, Boolean.valueOf(defaultValue));
+    }
+
+    @Then("the resolved boolean details value should be {string}, the variant should be {string}, and the reason should be {string}")
+    public void the_resolved_boolean_value_should_be_the_variant_should_be_and_the_reason_should_be(
+            String expectedValue,
+            String expectedVariant, String expectedReason) {
+        assertEquals(Boolean.valueOf(expectedValue), booleanFlagDetails.getValue());
+        assertEquals(expectedVariant, booleanFlagDetails.getVariant());
+        assertEquals(Reason.valueOf(expectedReason), booleanFlagDetails.getReason());
+    }
+
+    // string details
+    @When("a string flag with key {string} is evaluated with details and default value {string}")
+    public void a_string_flag_with_key_is_evaluated_with_details_and_default_value(String flagKey,
+            String defaultValue) {
+        this.stringFlagDetails = client.getStringDetails(flagKey, defaultValue);
+    }
+
+    @Then("the resolved string details value should be {string}, the variant should be {string}, and the reason should be {string}")
+    public void the_resolved_string_value_should_be_the_variant_should_be_and_the_reason_should_be(String expectedValue,
+            String expectedVariant, String expectedReason) {
+        assertEquals(expectedValue, this.stringFlagDetails.getValue());
+        assertEquals(expectedVariant, this.stringFlagDetails.getVariant());
+        assertEquals(Reason.valueOf(expectedReason), this.stringFlagDetails.getReason());
+    }
+
+    // integer details
+    @When("an integer flag with key {string} is evaluated with details and default value {int}")
+    public void an_integer_flag_with_key_is_evaluated_with_details_and_default_value(String flagKey, int defaultValue) {
+        this.intFlagDetails = client.getIntegerDetails(flagKey, defaultValue);
+    }
+
+    @Then("the resolved integer details value should be {int}, the variant should be {string}, and the reason should be {string}")
+    public void the_resolved_integer_value_should_be_the_variant_should_be_and_the_reason_should_be(int expectedValue,
+            String expectedVariant, String expectedReason) {
+        assertEquals(expectedValue, this.intFlagDetails.getValue());
+        assertEquals(expectedVariant, this.intFlagDetails.getVariant());
+        assertEquals(Reason.valueOf(expectedReason), this.intFlagDetails.getReason());
+    }
+
+    // float/double details
+    @When("a float flag with key {string} is evaluated with details and default value {double}")
+    public void a_float_flag_with_key_is_evaluated_with_details_and_default_value(String flagKey, double defaultValue) {
+        this.doubleFlagDetails = client.getDoubleDetails(flagKey, defaultValue);
+    }
+
+    @Then("the resolved float details value should be {double}, the variant should be {string}, and the reason should be {string}")
+    public void the_resolved_float_value_should_be_the_variant_should_be_and_the_reason_should_be(double expectedValue,
+            String expectedVariant, String expectedReason) {
+        assertEquals(expectedValue, this.doubleFlagDetails.getValue());
+        assertEquals(expectedVariant, this.doubleFlagDetails.getVariant());
+        assertEquals(Reason.valueOf(expectedReason), this.doubleFlagDetails.getReason());
+    }
+
+    // object details
+    @When("an object flag with key {string} is evaluated with details and a null default value")
+    public void an_object_flag_with_key_is_evaluated_with_details_and_a_null_default_value(String flagKey) {
+        this.objectFlagDetails = client.getObjectDetails(flagKey, new Value());
+    }
+
+    @Then("the resolved object details value should be contain fields {string}, {string}, and {string}, with values {string}, {string} and {int}, respectively")
+    public void the_resolved_object_value_should_be_contain_fields_and_with_values_and_respectively_again(
+            String boolField,
+            String stringField, String numberField, String boolValue, String stringValue, int numberValue) {
+        Structure structure = this.objectFlagDetails.getValue().asStructure();
+
+        assertEquals(Boolean.valueOf(boolValue), structure.asMap().get(boolField).asBoolean());
+        assertEquals(stringValue, structure.asMap().get(stringField).asString());
+        assertEquals(numberValue, structure.asMap().get(numberField).asInteger());
+    }
+
+    @Then("the variant should be {string}, and the reason should be {string}")
+    public void the_variant_should_be_and_the_reason_should_be(String expectedVariant, String expectedReason) {
+        assertEquals(expectedVariant, this.objectFlagDetails.getVariant());
+        assertEquals(Reason.valueOf(expectedReason), this.objectFlagDetails.getReason());
+    }
+
+    /*
+     * Context-aware evaluation
+     */
+
+    @When("context contains keys {string}, {string}, {string}, {string} with values {string}, {string}, {int}, {string}")
+    public void context_contains_keys_with_values(String field1, String field2, String field3, String field4,
+            String value1, String value2, Integer value3, String value4) {
+        this.context = new EvaluationContext()
+                .add(field1, value1)
+                .add(field2, value2)
+                .add(field3, value3)
+                .add(field4, Boolean.valueOf(value4));
+    }
+
+    @When("a flag with key {string} is evaluated with default value {string}")
+    public void an_a_flag_with_key_is_evaluated(String flagKey, String defaultValue) {
+        contextAwareFlagKey = flagKey;
+        contextAwareDefaultValue = defaultValue;
+        contextAwareValue = client.getStringValue(flagKey, contextAwareDefaultValue, context);
+
+    }
+
+    @Then("the resolved string response should be {string}")
+    public void the_resolved_string_response_should_be(String expected) {
+        assertEquals(expected, this.contextAwareValue);
+    }
+
+    @Then("the resolved flag value is {string} when the context is empty")
+    public void the_resolved_flag_value_is_when_the_context_is_empty(String expected) {
+        String emptyContextValue = client.getStringValue(contextAwareFlagKey, contextAwareDefaultValue,
+                new EvaluationContext());
+        assertEquals(expected, emptyContextValue);
+    }
+
+    /*
+     * Errors
+     */
+
+    // not found
+    @When("a non-existent string flag with key {string} is evaluated with details and a default value {string}")
+    public void a_non_existent_string_flag_with_key_is_evaluated_with_details_and_a_default_value(String flagKey,
+            String defaultValue) {
+        notFoundFlagKey = flagKey;
+        notFoundDefaultValue = defaultValue;
+        notFoundDetails = client.getStringDetails(notFoundFlagKey, notFoundDefaultValue);
+    }
+
+    @Then("then the default string value should be returned")
+    public void then_the_default_string_value_should_be_returned() {
+        assertEquals(notFoundDefaultValue, notFoundDetails.getValue());
+    }
+
+    @Then("the reason should indicate an error and the error code should indicate a missing flag with {string}")
+    public void the_reason_should_indicate_an_error_and_the_error_code_should_be_flag_not_found(String errorCode) {
+        assertEquals(Reason.ERROR, notFoundDetails.getReason());
+        assertTrue(notFoundDetails.getErrorCode().contains(errorCode));
+    }
+
+    // type mismatch
+    @When("a string flag with key {string} is evaluated as an integer, with details and a default value {int}")
+    public void a_string_flag_with_key_is_evaluated_as_an_integer_with_details_and_a_default_value(String flagKey,
+            int defaultValue) {
+        typeErrorFlagKey = flagKey;
+        typeErrorDefaultValue = defaultValue;
+        typeErrorDetails = client.getIntegerDetails(typeErrorFlagKey, typeErrorDefaultValue);
+    }
+
+    @Then("then the default integer value should be returned")
+    public void then_the_default_integer_value_should_be_returned() {
+        assertEquals(typeErrorDefaultValue, typeErrorDetails.getValue());
+    }
+
+    @Then("the reason should indicate an error and the error code should indicate a type mismatch with {string}")
+    public void the_reason_should_indicate_an_error_and_the_error_code_should_be_type_mismatch(String errorCode) {
+        assertEquals(Reason.ERROR, typeErrorDetails.getReason());
+        assertTrue(typeErrorDetails.getErrorCode().contains(errorCode));
+    }
+
+}

src/test/resources/features/.gitignore

@@ -0,0 +1 @@
+evaluation.feature

test-harness

@@ -0,0 +1 @@
+Subproject commit e7379cd0070f8907cacdc535184f8f626bf25e01