chore: add integration tests (#77)

* chore: add integration tests

Signed-off-by: Todd Baert <toddbaert@gmail.com>

* improve POM spacing

Signed-off-by: Todd Baert <toddbaert@gmail.com>

Signed-off-by: Todd Baert <toddbaert@gmail.com>
This commit is contained in:
Todd Baert 2022-09-20 09:48:34 -04:00 committed by GitHub
parent 2eec1a5519
commit f8d62eea2f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 437 additions and 4 deletions

View File

@ -9,6 +9,12 @@ permissions:
jobs:
build:
runs-on: ubuntu-latest
services:
flagd:
image: ghcr.io/open-feature/flagd-testbed:latest
ports:
- 8013:8013
steps:
- name: Check out the code
uses: actions/checkout@v3
@ -28,7 +34,7 @@ jobs:
${{ runner.os }}-maven-
- name: Build with Maven
run: mvn --batch-mode --update-snapshots verify
run: mvn --batch-mode --update-snapshots verify -P integration-test
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v2

3
.gitmodules vendored Normal file
View File

@ -0,0 +1,3 @@
[submodule "test-harness"]
path = test-harness
url = https://github.com/open-feature/test-harness

View File

@ -112,6 +112,12 @@ We hold regular meetings which you can see [here](https://github.com/open-featur
We are also present on the `#openfeature` channel in the [CNCF slack](https://slack.cncf.io/).
## Developing
### Integration tests
The continuous integration runs a set of [gherkin integration tests](https://github.com/open-feature/test-harness/blob/main/features/evaluation.feature) using [`flagd`](https://github.com/open-feature/flagd). These tests do not run with the default maven profile. If you'd like to run them locally, you can start the flagd testbed with `docker run -p 8013:8013 ghcr.io/open-feature/flagd-testbed:latest` and then run `mvn test -P integration-test`.
## Releasing
See [releasing](./docs/release.md).

124
pom.xml
View File

@ -11,6 +11,8 @@
<maven.compiler.source>1.8</maven.compiler.source>
<maven.compiler.target>${maven.compiler.source}</maven.compiler.target>
<junit.jupiter.version>5.8.1</junit.jupiter.version>
<!-- exclusion expression for integration tests -->
<testExclusions>**/integration/*.java</testExclusions>
</properties>
<name>OpenFeature Java SDK</name>
@ -46,7 +48,6 @@
<scope>provided</scope>
</dependency>
<dependency>
<!-- used so that lombok can generate suppressions for spotbugs. It needs to find it on the relevant classpath -->
<groupId>com.github.spotbugs</groupId>
@ -96,29 +97,81 @@
<version>${junit.jupiter.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-api</artifactId>
<version>${junit.jupiter.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-params</artifactId>
<version>${junit.jupiter.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.junit.platform</groupId>
<artifactId>junit-platform-suite</artifactId>
<version>1.8.1</version>
<version>1.9.0</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.cucumber</groupId>
<artifactId>cucumber-java</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.cucumber</groupId>
<artifactId>cucumber-junit-platform-engine</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>31.1-jre</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>dev.openfeature.contrib.providers</groupId>
<artifactId>flagd</artifactId>
<version>0.3.2</version>
<scope>test</scope>
</dependency>
</dependencies>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>io.cucumber</groupId>
<artifactId>cucumber-bom</artifactId>
<version>7.5.0</version>
<type>pom</type>
<scope>import</scope>
</dependency>
<dependency>
<groupId>org.junit</groupId>
<artifactId>junit-bom</artifactId>
<version>5.9.0</version>
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>
<build>
<plugins>
<plugin>
<artifactId>maven-dependency-plugin</artifactId>
<version>3.3.0</version>
@ -137,16 +190,21 @@
<ignoredUnusedDeclaredDependency>org.junit*</ignoredUnusedDeclaredDependency>
</ignoredUnusedDeclaredDependencies>
<ignoredDependencies>
<ignoredDependency>com.google.guava*</ignoredDependency>
<ignoredDependency>io.cucumber*</ignoredDependency>
<ignoredDependency>org.junit*</ignoredDependency>
<ignoredDependency>com.google.code.findbugs*</ignoredDependency>
<ignoredDependency>com.github.spotbugs*</ignoredDependency>
<ignoredDependency>uk.org.lidalia:lidalia-slf4j-ext:*</ignoredDependency>
</ignoredDependencies>
</configuration>
</plugin>
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.8.1</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
@ -155,6 +213,10 @@
<argLine>
${surefireArgLine}
</argLine>
<excludes>
<!-- tests to exclude -->
<exclude>${testExclusions}</exclude>
</excludes>
</configuration>
</plugin>
@ -224,7 +286,7 @@
</rule>
</rules>
</configuration>
</execution>
</execution>
</executions>
</plugin>
@ -257,6 +319,7 @@
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
@ -372,6 +435,61 @@
</plugins>
</build>
<profiles>
<profile>
<!-- this profile handles running the flagd integration tests -->
<id>integration-test</id>
<properties>
<!-- run the integration tests by clearing the exclusions -->
<testExclusions></testExclusions>
</properties>
<build>
<plugins>
<!-- pull the gherkin tests as a git submodule -->
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
<version>3.1.0</version>
<executions>
<execution>
<id>update-test-harness-submodule</id>
<phase>validate</phase>
<goals>
<goal>exec</goal>
</goals>
<configuration>
<!-- run: git submodule update \-\-init \-\-recursive -->
<executable>git</executable>
<arguments>
<argument>submodule</argument>
<argument>update</argument>
<argument>--init</argument>
<argument>--recursive</argument>
</arguments>
</configuration>
</execution>
<execution>
<id>copy-gherkin-tests</id>
<phase>validate</phase>
<goals>
<goal>exec</goal>
</goals>
<configuration>
<!-- run: cp test-harness/features/evaluation.feature src/test/resources/features/ -->
<executable>cp</executable>
<arguments>
<argument>test-harness/features/evaluation.feature</argument>
<argument>src/test/resources/features/</argument>
</arguments>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
</profiles>
<distributionManagement>
<snapshotRepository>
<id>ossrh</id>

View File

@ -0,0 +1,16 @@
package dev.openfeature.javasdk.integration;
import org.junit.platform.suite.api.ConfigurationParameter;
import org.junit.platform.suite.api.IncludeEngines;
import org.junit.platform.suite.api.SelectClasspathResource;
import org.junit.platform.suite.api.Suite;
import static io.cucumber.junit.platform.engine.Constants.PLUGIN_PROPERTY_NAME;
@Suite
@IncludeEngines("cucumber")
@SelectClasspathResource("features")
@ConfigurationParameter(key = PLUGIN_PROPERTY_NAME, value = "pretty")
public class RunCucumberTest {
}

View File

@ -0,0 +1,282 @@
package dev.openfeature.javasdk.integration;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assumptions.assumeFalse;
import dev.openfeature.contrib.providers.flagd.FlagdProvider;
import dev.openfeature.javasdk.Client;
import dev.openfeature.javasdk.ErrorCode;
import dev.openfeature.javasdk.EvaluationContext;
import dev.openfeature.javasdk.FlagEvaluationDetails;
import dev.openfeature.javasdk.OpenFeatureAPI;
import dev.openfeature.javasdk.Reason;
import dev.openfeature.javasdk.Structure;
import dev.openfeature.javasdk.Value;
import io.cucumber.java.BeforeAll;
import io.cucumber.java.en.Then;
import io.cucumber.java.en.When;
public class StepDefinitions {
private static Client client;
private boolean booleanFlagValue;
private String stringFlagValue;
private int intFlagValue;
private double doubleFlagValue;
private Value objectFlagValue;
private FlagEvaluationDetails<Boolean> booleanFlagDetails;
private FlagEvaluationDetails<String> stringFlagDetails;
private FlagEvaluationDetails<Integer> intFlagDetails;
private FlagEvaluationDetails<Double> doubleFlagDetails;
private FlagEvaluationDetails<Value> objectFlagDetails;
private String contextAwareFlagKey;
private String contextAwareDefaultValue;
private EvaluationContext context;
private String contextAwareValue;
private String notFoundFlagKey;
private String notFoundDefaultValue;
private FlagEvaluationDetails<String> notFoundDetails;
private String typeErrorFlagKey;
private int typeErrorDefaultValue;
private FlagEvaluationDetails<Integer> typeErrorDetails;
@BeforeAll()
public static void setup() {
OpenFeatureAPI.getInstance().setProvider(new FlagdProvider());
client = OpenFeatureAPI.getInstance().getClient();
}
/*
* Basic evaluation
*/
// boolean value
@When("a boolean flag with key {string} is evaluated with default value {string}")
public void a_boolean_flag_with_key_boolean_flag_is_evaluated_with_default_value_false(String flagKey,
String defaultValue) {
this.booleanFlagValue = client.getBooleanValue(flagKey, Boolean.valueOf(defaultValue));
}
@Then("the resolved boolean value should be {string}")
public void the_resolved_boolean_value_should_be_true(String expected) {
assertEquals(Boolean.valueOf(expected), this.booleanFlagValue);
}
// string value
@When("a string flag with key {string} is evaluated with default value {string}")
public void a_string_flag_with_key_is_evaluated_with_default_value(String flagKey, String defaultValue) {
this.stringFlagValue = client.getStringValue(flagKey, defaultValue);
}
@Then("the resolved string value should be {string}")
public void the_resolved_string_value_should_be(String expected) {
assertEquals(expected, this.stringFlagValue);
}
// integer value
@When("an integer flag with key {string} is evaluated with default value {int}")
public void an_integer_flag_with_key_is_evaluated_with_default_value(String flagKey, Integer defaultValue) {
this.intFlagValue = client.getIntegerValue(flagKey, defaultValue);
}
@Then("the resolved integer value should be {int}")
public void the_resolved_integer_value_should_be(int expected) {
assertEquals(expected, this.intFlagValue);
}
// float/double value
@When("a float flag with key {string} is evaluated with default value {double}")
public void a_float_flag_with_key_is_evaluated_with_default_value(String flagKey, double defaultValue) {
this.doubleFlagValue = client.getDoubleValue(flagKey, defaultValue);
}
@Then("the resolved float value should be {double}")
public void the_resolved_float_value_should_be(double expected) {
assertEquals(expected, this.doubleFlagValue);
}
// object value
@When("an object flag with key {string} is evaluated with a null default value")
public void an_object_flag_with_key_is_evaluated_with_a_null_default_value(String flagKey) {
this.objectFlagValue = client.getObjectValue(flagKey, new Value());
}
@Then("the resolved object value should be contain fields {string}, {string}, and {string}, with values {string}, {string} and {int}, respectively")
public void the_resolved_object_value_should_be_contain_fields_and_with_values_and_respectively(String boolField,
String stringField, String numberField, String boolValue, String stringValue, int numberValue) {
Structure structure = this.objectFlagValue.asStructure();
assertEquals(Boolean.valueOf(boolValue), structure.asMap().get(boolField).asBoolean());
assertEquals(stringValue, structure.asMap().get(stringField).asString());
assertEquals(numberValue, structure.asMap().get(numberField).asInteger());
}
/*
* Detailed evaluation
*/
// boolean details
@When("a boolean flag with key {string} is evaluated with details and default value {string}")
public void a_boolean_flag_with_key_is_evaluated_with_details_and_default_value(String flagKey,
String defaultValue) {
this.booleanFlagDetails = client.getBooleanDetails(flagKey, Boolean.valueOf(defaultValue));
}
@Then("the resolved boolean details value should be {string}, the variant should be {string}, and the reason should be {string}")
public void the_resolved_boolean_value_should_be_the_variant_should_be_and_the_reason_should_be(
String expectedValue,
String expectedVariant, String expectedReason) {
assertEquals(Boolean.valueOf(expectedValue), booleanFlagDetails.getValue());
assertEquals(expectedVariant, booleanFlagDetails.getVariant());
assertEquals(Reason.valueOf(expectedReason), booleanFlagDetails.getReason());
}
// string details
@When("a string flag with key {string} is evaluated with details and default value {string}")
public void a_string_flag_with_key_is_evaluated_with_details_and_default_value(String flagKey,
String defaultValue) {
this.stringFlagDetails = client.getStringDetails(flagKey, defaultValue);
}
@Then("the resolved string details value should be {string}, the variant should be {string}, and the reason should be {string}")
public void the_resolved_string_value_should_be_the_variant_should_be_and_the_reason_should_be(String expectedValue,
String expectedVariant, String expectedReason) {
assertEquals(expectedValue, this.stringFlagDetails.getValue());
assertEquals(expectedVariant, this.stringFlagDetails.getVariant());
assertEquals(Reason.valueOf(expectedReason), this.stringFlagDetails.getReason());
}
// integer details
@When("an integer flag with key {string} is evaluated with details and default value {int}")
public void an_integer_flag_with_key_is_evaluated_with_details_and_default_value(String flagKey, int defaultValue) {
this.intFlagDetails = client.getIntegerDetails(flagKey, defaultValue);
}
@Then("the resolved integer details value should be {int}, the variant should be {string}, and the reason should be {string}")
public void the_resolved_integer_value_should_be_the_variant_should_be_and_the_reason_should_be(int expectedValue,
String expectedVariant, String expectedReason) {
assertEquals(expectedValue, this.intFlagDetails.getValue());
assertEquals(expectedVariant, this.intFlagDetails.getVariant());
assertEquals(Reason.valueOf(expectedReason), this.intFlagDetails.getReason());
}
// float/double details
@When("a float flag with key {string} is evaluated with details and default value {double}")
public void a_float_flag_with_key_is_evaluated_with_details_and_default_value(String flagKey, double defaultValue) {
this.doubleFlagDetails = client.getDoubleDetails(flagKey, defaultValue);
}
@Then("the resolved float details value should be {double}, the variant should be {string}, and the reason should be {string}")
public void the_resolved_float_value_should_be_the_variant_should_be_and_the_reason_should_be(double expectedValue,
String expectedVariant, String expectedReason) {
assertEquals(expectedValue, this.doubleFlagDetails.getValue());
assertEquals(expectedVariant, this.doubleFlagDetails.getVariant());
assertEquals(Reason.valueOf(expectedReason), this.doubleFlagDetails.getReason());
}
// object details
@When("an object flag with key {string} is evaluated with details and a null default value")
public void an_object_flag_with_key_is_evaluated_with_details_and_a_null_default_value(String flagKey) {
this.objectFlagDetails = client.getObjectDetails(flagKey, new Value());
}
@Then("the resolved object details value should be contain fields {string}, {string}, and {string}, with values {string}, {string} and {int}, respectively")
public void the_resolved_object_value_should_be_contain_fields_and_with_values_and_respectively_again(
String boolField,
String stringField, String numberField, String boolValue, String stringValue, int numberValue) {
Structure structure = this.objectFlagDetails.getValue().asStructure();
assertEquals(Boolean.valueOf(boolValue), structure.asMap().get(boolField).asBoolean());
assertEquals(stringValue, structure.asMap().get(stringField).asString());
assertEquals(numberValue, structure.asMap().get(numberField).asInteger());
}
@Then("the variant should be {string}, and the reason should be {string}")
public void the_variant_should_be_and_the_reason_should_be(String expectedVariant, String expectedReason) {
assertEquals(expectedVariant, this.objectFlagDetails.getVariant());
assertEquals(Reason.valueOf(expectedReason), this.objectFlagDetails.getReason());
}
/*
* Context-aware evaluation
*/
@When("context contains keys {string}, {string}, {string}, {string} with values {string}, {string}, {int}, {string}")
public void context_contains_keys_with_values(String field1, String field2, String field3, String field4,
String value1, String value2, Integer value3, String value4) {
this.context = new EvaluationContext()
.add(field1, value1)
.add(field2, value2)
.add(field3, value3)
.add(field4, Boolean.valueOf(value4));
}
@When("a flag with key {string} is evaluated with default value {string}")
public void an_a_flag_with_key_is_evaluated(String flagKey, String defaultValue) {
contextAwareFlagKey = flagKey;
contextAwareDefaultValue = defaultValue;
contextAwareValue = client.getStringValue(flagKey, contextAwareDefaultValue, context);
}
@Then("the resolved string response should be {string}")
public void the_resolved_string_response_should_be(String expected) {
assertEquals(expected, this.contextAwareValue);
}
@Then("the resolved flag value is {string} when the context is empty")
public void the_resolved_flag_value_is_when_the_context_is_empty(String expected) {
String emptyContextValue = client.getStringValue(contextAwareFlagKey, contextAwareDefaultValue,
new EvaluationContext());
assertEquals(expected, emptyContextValue);
}
/*
* Errors
*/
// not found
@When("a non-existent string flag with key {string} is evaluated with details and a default value {string}")
public void a_non_existent_string_flag_with_key_is_evaluated_with_details_and_a_default_value(String flagKey,
String defaultValue) {
notFoundFlagKey = flagKey;
notFoundDefaultValue = defaultValue;
notFoundDetails = client.getStringDetails(notFoundFlagKey, notFoundDefaultValue);
}
@Then("then the default string value should be returned")
public void then_the_default_string_value_should_be_returned() {
assertEquals(notFoundDefaultValue, notFoundDetails.getValue());
}
@Then("the reason should indicate an error and the error code should indicate a missing flag with {string}")
public void the_reason_should_indicate_an_error_and_the_error_code_should_be_flag_not_found(String errorCode) {
assertEquals(Reason.ERROR, notFoundDetails.getReason());
assertTrue(notFoundDetails.getErrorCode().contains(errorCode));
}
// type mismatch
@When("a string flag with key {string} is evaluated as an integer, with details and a default value {int}")
public void a_string_flag_with_key_is_evaluated_as_an_integer_with_details_and_a_default_value(String flagKey,
int defaultValue) {
typeErrorFlagKey = flagKey;
typeErrorDefaultValue = defaultValue;
typeErrorDetails = client.getIntegerDetails(typeErrorFlagKey, typeErrorDefaultValue);
}
@Then("then the default integer value should be returned")
public void then_the_default_integer_value_should_be_returned() {
assertEquals(typeErrorDefaultValue, typeErrorDetails.getValue());
}
@Then("the reason should indicate an error and the error code should indicate a type mismatch with {string}")
public void the_reason_should_indicate_an_error_and_the_error_code_should_be_type_mismatch(String errorCode) {
assertEquals(Reason.ERROR, typeErrorDetails.getReason());
assertTrue(typeErrorDetails.getErrorCode().contains(errorCode));
}
}

View File

@ -0,0 +1 @@
evaluation.feature

View File

1
test-harness Submodule

@ -0,0 +1 @@
Subproject commit e7379cd0070f8907cacdc535184f8f626bf25e01