Run conformance Cucumber tests (#474)

* Run conformance Cucumber tests

Signed-off-by: Sergei Egorov <segorov@pivotal.io>

* fix CircleCI job name

Signed-off-by: Sergei Egorov <segorov@pivotal.io>

* fix integration testing step

Signed-off-by: Sergei Egorov <segorov@pivotal.io>

* checkout submodules

Signed-off-by: Sergei Egorov <segorov@pivotal.io>

* Update to cloudevents/conformance

Signed-off-by: Sergei Egorov <segorov@pivotal.io>
This commit is contained in:
Sergei Egorov 2020-04-28 16:55:34 +02:00 committed by GitHub
parent 77c056979e
commit c30f66a8a6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 264 additions and 1 deletions

View File

@ -36,6 +36,7 @@ jobs:
steps:
- checkout
- run: git submodule sync && git submodule update --init
- run: mkdir -p $TEST_RESULTS
- run: mkdir -p $COVERAGE_RESULTS
@ -63,6 +64,10 @@ jobs:
name: Run integrations tests
command: |
TEST_AMQP_URL=amqp://localhost/test ./hack/integration-test.sh | tee ${TEST_RESULTS}/go-integration-test.out
- run:
name: Run conformance tests
command: |
./hack/conformance-test.sh | tee ${TEST_RESULTS}/go-conformance-test.out
- run:
name: Generate coverage report

4
.gitmodules vendored Normal file
View File

@ -0,0 +1,4 @@
[submodule "conformance"]
path = conformance
url = https://github.com/cloudevents/conformance.git
branch = master

1
conformance Submodule

@ -0,0 +1 @@
Subproject commit 6366de11ed8297b6ad228e04869eadfd2a003266

14
hack/conformance-test.sh Executable file
View File

@ -0,0 +1,14 @@
#!/usr/bin/env bash
set -o errexit
set -o nounset
set -o pipefail
# v2 only
pushd ./v2/test/conformance/
go test -v -timeout 15s
# Remove test only deps.
go mod tidy
popd

View File

@ -14,7 +14,7 @@ if [ ! -f ./coverage.txt ]; then
echo 'mode: atomic' > ./coverage.txt
fi
COVERPKG=$(go list ./... | grep -v /vendor | tr "\n" ",")
for gomodule in $(go list ./test/... | grep -v /cmd | grep -v /vendor)
for gomodule in $(go list ./test/integration/... | grep -v /cmd | grep -v /vendor)
do
go test -v -parallel 1 -timeout 60s -covermode=atomic -coverprofile=coverage.tmp -coverpkg "$COVERPKG" "$gomodule" 2>&1 | sed 's/ of statements in.*//; /warning: no packages being tested depend on matches for pattern /d'
tail -n +2 coverage.tmp >> ./coverage.txt

View File

@ -6,6 +6,8 @@ require (
github.com/Azure/go-amqp v0.12.7
github.com/Shopify/sarama v1.19.0
github.com/cloudevents/sdk-go v1.1.2
github.com/cucumber/godog v0.9.0
github.com/cucumber/messages-go/v10 v10.0.3
github.com/google/go-cmp v0.4.0
github.com/google/uuid v1.1.1
github.com/gorilla/mux v1.6.2

View File

@ -39,6 +39,7 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF
github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 h1:EFSB7Zo9Eg91v7MJPVsifUysc/wPdN+NOnVe6bWbdBM=
github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQhVx52RsWOnlkpikZr01T/yAVN2gn0861vByNg=
github.com/aslakhellesoy/gox v1.0.100/go.mod h1:AJl542QsKKG96COVsv0N74HHzVQgDIQPceVUh1aeU2M=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
@ -51,6 +52,14 @@ github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/cloudevents/sdk-go v1.1.2 h1:mg/7d+BzubBPrPpH1bdeF85BQZYV85j7Ljqat3+m+qE=
github.com/cloudevents/sdk-go v1.1.2/go.mod h1:ss+jWJ88wypiewnPEzChSBzTYXGpdcILoN9YHk8uhTQ=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/cucumber/gherkin-go/v11 v11.0.0 h1:cwVwN1Qn2VRSfHZNLEh5x00tPBmZcjATBWDpxsR5Xug=
github.com/cucumber/gherkin-go/v11 v11.0.0/go.mod h1:CX33k2XU2qog4e+TFjOValoq6mIUq0DmVccZs238R9w=
github.com/cucumber/godog v0.9.0 h1:QOb8wyC7f+FVFXzY3RdgowwJUb4WeJfqbnQqaH4jp+A=
github.com/cucumber/godog v0.9.0/go.mod h1:roWCHkpeK6UTOyIRRl7IR+fgfBeZ4vZR7OSq2J/NbM4=
github.com/cucumber/messages-go/v10 v10.0.1/go.mod h1:kA5T38CBlBbYLU12TIrJ4fk4wSkVVOgyh7Enyy8WnSg=
github.com/cucumber/messages-go/v10 v10.0.3 h1:m/9SD/K/A15WP7i1aemIv7cwvUw+viS51Ui5HBw1cdE=
github.com/cucumber/messages-go/v10 v10.0.3/go.mod h1:9jMZ2Y8ZxjLY6TG2+x344nt5rXstVVDYSdS5ySfI1WY=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@ -74,6 +83,8 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9
github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs=
github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/gofrs/uuid v3.2.0+incompatible h1:y12jRkkFxsd7GpqdSZ+/KCs/fJbqpEXSGd4+jfEaewE=
github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE=
@ -122,6 +133,7 @@ github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iP
github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs=
github.com/hashicorp/go-uuid v1.0.0 h1:RS8zrF7PhGwyNPOtxSClXXj9HA8feRnJzgnI1RJCSnM=
github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
github.com/hashicorp/go-version v1.0.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU=
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
@ -146,12 +158,15 @@ github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfn
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/lib/pq v1.3.0 h1:/qkRGz8zljWiDcFvgpwUpwIAPu3r07TDvs3Rws+o/pU=
github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/lightstep/tracecontext.go v0.0.0-20181129014701-1757c391b1ac h1:+2b6iGRJe3hvV/yVXrd41yVEjxuFHxasJqDhkIjS4gk=
github.com/lightstep/tracecontext.go v0.0.0-20181129014701-1757c391b1ac/go.mod h1:Frd2bnT3w5FB5q49ENTfVlztJES+1k/7lyWX2+9gq/M=
github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg=
github.com/nats-io/jwt v0.3.2 h1:+RB5hMpXUUA2dfxuhBTEkMOrYmM+gKIZYS1KjSostMI=
@ -171,6 +186,8 @@ github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
github.com/nats-io/stan.go v0.6.0 h1:26IJPeykh88d8KVLT4jJCIxCyUBOC5/IQup8oWD/QYY=
github.com/nats-io/stan.go v0.6.0/go.mod h1:eIcD5bi3pqbHT/xIIvXMwvzXYElgouBvaVRftaE+eac=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs=
github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
@ -345,6 +362,8 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=

View File

@ -0,0 +1,74 @@
package conformance
import (
"encoding/json"
"fmt"
"time"
"github.com/cloudevents/sdk-go/v2/event"
"github.com/cucumber/godog"
messages "github.com/cucumber/messages-go/v10"
"github.com/google/go-cmp/cmp"
)
var currentEvent *event.Event
func CloudEventsFeatureContext(s *godog.Suite) {
s.BeforeScenario(func(message *messages.Pickle) {
currentEvent = nil
})
s.Step(`^the attributes are:$`, func(attributes *messages.PickleStepArgument_PickleTable) error {
for _, row := range attributes.Rows {
key := row.Cells[0].Value
value := row.Cells[1].Value
var actual string
switch key {
case "key":
// ignore the header
continue
case "specversion":
actual = currentEvent.SpecVersion()
case "id":
actual = currentEvent.ID()
case "type":
actual = currentEvent.Type()
case "source":
actual = currentEvent.Source()
case "time":
actual = currentEvent.Time().Format(time.RFC3339)
case "datacontenttype":
actual = currentEvent.DataContentType()
default:
return fmt.Errorf("Unknown key '%s'", key)
}
if diff := cmp.Diff(value, actual); diff != "" {
return fmt.Errorf("unexpected '%s' (-want, +got) = %v", key, diff)
}
}
return nil
})
s.Step(`^the data is equal to the following JSON:$`, func(jsonData *messages.PickleStepArgument_PickleDocString) error {
actualBytes := currentEvent.Data()
var expectedJSONAsInterface, actualJSONAsInterface interface{}
if err := json.Unmarshal([]byte(jsonData.Content), &expectedJSONAsInterface); err != nil {
return fmt.Errorf("Expected value ('%s') is not valid json.\nJSON parsing error: '%s'", jsonData.Content, err.Error())
}
if err := json.Unmarshal(actualBytes, &actualJSONAsInterface); err != nil {
return fmt.Errorf("Input ('%s') needs to be valid json.\nJSON parsing error: '%s'", string(actualBytes), err.Error())
}
if diff := cmp.Diff(expectedJSONAsInterface, actualJSONAsInterface); diff != "" {
return fmt.Errorf("unexpected (-want, +got) = %v", diff)
}
return nil
})
}

View File

@ -0,0 +1,51 @@
package conformance
import (
"bufio"
"context"
"strings"
nethttp "net/http"
"github.com/cloudevents/sdk-go/v2/binding"
"github.com/cloudevents/sdk-go/v2/protocol/http"
"github.com/cucumber/godog"
messages "github.com/cucumber/messages-go/v10"
)
var request *nethttp.Request
func HTTPFeatureContext(s *godog.Suite) {
s.BeforeScenario(func(message *messages.Pickle) {
request = nil
})
s.Step(`^HTTP Protocol Binding is supported$`, func() error {
return nil
})
s.Step(`^an HTTP request$`, func(rawRequest *messages.PickleStepArgument_PickleDocString) error {
parsedRequest, err := nethttp.ReadRequest(bufio.NewReader(strings.NewReader(rawRequest.Content)))
if err != nil {
return err
}
request = parsedRequest
return nil
})
s.Step(`^parsed as HTTP request$`, func() error {
message := http.NewMessageFromHttpRequest(request)
event, err := binding.ToEvent(context.TODO(), message)
if err != nil {
return err
}
currentEvent = event
return err
})
}

View File

@ -0,0 +1,55 @@
package conformance
import (
"context"
"github.com/Shopify/sarama"
"github.com/cloudevents/sdk-go/v2/binding"
"github.com/cloudevents/sdk-go/v2/protocol/kafka_sarama"
"github.com/cucumber/godog"
messages "github.com/cucumber/messages-go/v10"
)
var consumerMessage *sarama.ConsumerMessage
func KafkaFeatureContext(s *godog.Suite) {
s.BeforeScenario(func(message *messages.Pickle) {
consumerMessage = nil
})
s.Step(`^Kafka Protocol Binding is supported$`, func() error {
return nil
})
s.Step(`^a Kafka message with payload:$`, func(payload *messages.PickleStepArgument_PickleDocString) error {
consumerMessage = &sarama.ConsumerMessage{
Value: []byte(payload.Content),
}
return nil
})
s.Step(`^Kafka headers:$`, func(headers *messages.PickleStepArgument_PickleTable) error {
consumerMessage.Headers = make([]*sarama.RecordHeader, len(headers.Rows))
for i, row := range headers.Rows {
var key = row.Cells[0].Value
var value = row.Cells[1].Value
consumerMessage.Headers[i] = &sarama.RecordHeader{Key: []byte(key), Value: []byte(value)}
}
return nil
})
s.Step(`^parsed as Kafka message$`, func() error {
message := kafka_sarama.NewMessageFromConsumerMessage(consumerMessage)
event, err := binding.ToEvent(context.TODO(), message)
if err != nil {
return err
}
currentEvent = event
return nil
})
}

View File

@ -0,0 +1,38 @@
package conformance
import (
"flag"
"os"
"testing"
"github.com/cucumber/godog"
"github.com/cucumber/godog/colors"
)
var opt = godog.Options{
Output: colors.Colored(os.Stdout),
}
func TestMain(m *testing.M) {
flag.Parse()
if len(flag.Args()) > 0 {
opt.Paths = flag.Args()
} else {
opt.Paths = []string{
"../../../conformance/features/",
}
}
opt.Format = "pretty"
status := godog.RunWithOptions("CloudEvents", func(s *godog.Suite) {
CloudEventsFeatureContext(s)
HTTPFeatureContext(s)
KafkaFeatureContext(s)
}, opt)
if st := m.Run(); st > status {
status = st
}
os.Exit(status)
}