Make testbed usable from other repos (#449)

1. Extract perf tests to "Scenario" funcs which are parametrized and use the
   scenarios in Test funcs in this repo. This allows the same scenarios to
   be used in other repos (such as "contrib"). The intent is to perform the same
   scenarios with other protocols defined in "contrib".

2. Introduce helper funcs like DoTestMain that make it easy to use the testbed
   from other repos.

3. Make testutils package public to allow it to be used from testbeds in other repos.

4. Fixed broken targets in makefile.

Makefile

@@ -34,10 +34,10 @@ all-pkgs:
 all-srcs:
 	@echo $(ALL_SRC) | tr ' ' '\n' | sort
 
-.DEFAULT_GOAL := addlicense-fmt-impi-vet-lint-goimports-misspell-staticcheck-test
+.DEFAULT_GOAL := all
 
-.PHONY: addlicense-fmt-impi-vet-lint-goimports-misspell-staticcheck-test
-addlicense-fmt-impi-vet-lint-goimports-misspell-staticcheck-test: addlicense fmt impi vet lint goimports misspell staticcheck test
+.PHONY: all
+all: addlicense impi lint misspell test otelcol
 
 .PHONY: e2e-test
 e2e-test: otelcol
@@ -52,7 +52,7 @@ benchmark:
 	$(GOTEST) -bench=. -run=notests $(ALL_PKGS)
 
 .PHONY: travis-ci
-travis-ci: impi lint misspell test-with-cover otelcol
+travis-ci: all test-with-cover
 	$(MAKE) -C testbed install-tools
 	$(MAKE) -C testbed runtests
 

exporter/opencensusexporter/factory_test.go

@@ -27,9 +27,9 @@ import (
 	"github.com/open-telemetry/opentelemetry-collector/config/configcheck"
 	"github.com/open-telemetry/opentelemetry-collector/config/configgrpc"
 	"github.com/open-telemetry/opentelemetry-collector/exporter/exportertest"
-	"github.com/open-telemetry/opentelemetry-collector/internal/testutils"
 	"github.com/open-telemetry/opentelemetry-collector/receiver/opencensusreceiver"
 	"github.com/open-telemetry/opentelemetry-collector/receiver/receivertest"
+	"github.com/open-telemetry/opentelemetry-collector/testutils"
 )
 
 func TestCreateDefaultConfig(t *testing.T) {

exporter/zipkinexporter/zipkin_test.go

@@ -33,10 +33,10 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/open-telemetry/opentelemetry-collector/consumer"
-	"github.com/open-telemetry/opentelemetry-collector/internal/testutils"
 	"github.com/open-telemetry/opentelemetry-collector/processor"
 	"github.com/open-telemetry/opentelemetry-collector/receiver/receivertest"
 	"github.com/open-telemetry/opentelemetry-collector/receiver/zipkinreceiver"
+	"github.com/open-telemetry/opentelemetry-collector/testutils"
 	"github.com/open-telemetry/opentelemetry-collector/translator/trace/zipkin"
 )
 

extension/healthcheckextension/factory_test.go

@@ -24,7 +24,7 @@ import (
 
 	"github.com/open-telemetry/opentelemetry-collector/config/configcheck"
 	"github.com/open-telemetry/opentelemetry-collector/config/configmodels"
-	"github.com/open-telemetry/opentelemetry-collector/internal/testutils"
+	"github.com/open-telemetry/opentelemetry-collector/testutils"
 )
 
 func TestFactory_Type(t *testing.T) {

extension/healthcheckextension/healthcheckextension_test.go

@@ -26,7 +26,7 @@ import (
 	"go.uber.org/zap"
 
 	"github.com/open-telemetry/opentelemetry-collector/extension/extensiontest"
-	"github.com/open-telemetry/opentelemetry-collector/internal/testutils"
+	"github.com/open-telemetry/opentelemetry-collector/testutils"
 )
 
 func TestHealthCheckExtensionUsage(t *testing.T) {

extension/pprofextension/factory_test.go

@@ -24,7 +24,7 @@ import (
 
 	"github.com/open-telemetry/opentelemetry-collector/config/configcheck"
 	"github.com/open-telemetry/opentelemetry-collector/config/configmodels"
-	"github.com/open-telemetry/opentelemetry-collector/internal/testutils"
+	"github.com/open-telemetry/opentelemetry-collector/testutils"
 )
 
 func TestFactory_Type(t *testing.T) {

extension/pprofextension/pprofextension_test.go

@@ -24,7 +24,7 @@ import (
 	"go.uber.org/zap"
 
 	"github.com/open-telemetry/opentelemetry-collector/extension/extensiontest"
-	"github.com/open-telemetry/opentelemetry-collector/internal/testutils"
+	"github.com/open-telemetry/opentelemetry-collector/testutils"
 )
 
 func TestPerformanceProfilerExtensionUsage(t *testing.T) {

extension/zpagesextension/factory_test.go

@@ -24,7 +24,7 @@ import (
 
 	"github.com/open-telemetry/opentelemetry-collector/config/configcheck"
 	"github.com/open-telemetry/opentelemetry-collector/config/configmodels"
-	"github.com/open-telemetry/opentelemetry-collector/internal/testutils"
+	"github.com/open-telemetry/opentelemetry-collector/testutils"
 )
 
 func TestFactory_Type(t *testing.T) {

extension/zpagesextension/zpagesextension_test.go

@@ -24,7 +24,7 @@ import (
 	"go.uber.org/zap"
 
 	"github.com/open-telemetry/opentelemetry-collector/extension/extensiontest"
-	"github.com/open-telemetry/opentelemetry-collector/internal/testutils"
+	"github.com/open-telemetry/opentelemetry-collector/testutils"
 )
 
 func TestZPagesExtensionUsage(t *testing.T) {

receiver/jaegerreceiver/jaeger_agent_test.go

@@ -33,8 +33,8 @@ import (
 	"github.com/open-telemetry/opentelemetry-collector/consumer/consumerdata"
 	"github.com/open-telemetry/opentelemetry-collector/exporter/exportertest"
 	"github.com/open-telemetry/opentelemetry-collector/internal"
-	"github.com/open-telemetry/opentelemetry-collector/internal/testutils"
 	"github.com/open-telemetry/opentelemetry-collector/receiver/receivertest"
+	"github.com/open-telemetry/opentelemetry-collector/testutils"
 )
 
 func TestJaegerAgentUDP_ThriftCompact_6831(t *testing.T) {

receiver/opencensusreceiver/factory_test.go

@@ -26,9 +26,9 @@ import (
 	"github.com/open-telemetry/opentelemetry-collector/config/configcheck"
 	"github.com/open-telemetry/opentelemetry-collector/config/configmodels"
 	"github.com/open-telemetry/opentelemetry-collector/exporter/exportertest"
-	"github.com/open-telemetry/opentelemetry-collector/internal/testutils"
 	"github.com/open-telemetry/opentelemetry-collector/receiver"
 	"github.com/open-telemetry/opentelemetry-collector/receiver/receivertest"
+	"github.com/open-telemetry/opentelemetry-collector/testutils"
 )
 
 func TestCreateDefaultConfig(t *testing.T) {

receiver/opencensusreceiver/opencensus_test.go

@@ -35,8 +35,8 @@ import (
 	"github.com/open-telemetry/opentelemetry-collector/consumer/consumerdata"
 	"github.com/open-telemetry/opentelemetry-collector/exporter/exportertest"
 	"github.com/open-telemetry/opentelemetry-collector/internal"
-	"github.com/open-telemetry/opentelemetry-collector/internal/testutils"
 	"github.com/open-telemetry/opentelemetry-collector/receiver/receivertest"
+	"github.com/open-telemetry/opentelemetry-collector/testutils"
 )
 
 // TODO(ccaraman): Migrate tests to use assert for validating functionality.

receiver/zipkinreceiver/trace_receiver_test.go

@@ -39,10 +39,10 @@ import (
 	"github.com/open-telemetry/opentelemetry-collector/exporter/exportertest"
 	"github.com/open-telemetry/opentelemetry-collector/exporter/zipkinexporter"
 	"github.com/open-telemetry/opentelemetry-collector/internal"
-	"github.com/open-telemetry/opentelemetry-collector/internal/testutils"
 	"github.com/open-telemetry/opentelemetry-collector/oterr"
 	"github.com/open-telemetry/opentelemetry-collector/receiver"
 	"github.com/open-telemetry/opentelemetry-collector/receiver/receivertest"
+	"github.com/open-telemetry/opentelemetry-collector/testutils"
 	"github.com/open-telemetry/opentelemetry-collector/translator/trace/zipkin"
 )
 

service/service_test.go

@@ -28,7 +28,7 @@ import (
 	"github.com/open-telemetry/opentelemetry-collector/config/configmodels"
 	"github.com/open-telemetry/opentelemetry-collector/defaults"
 	"github.com/open-telemetry/opentelemetry-collector/extension"
-	"github.com/open-telemetry/opentelemetry-collector/internal/testutils"
+	"github.com/open-telemetry/opentelemetry-collector/testutils"
 )
 
 func TestApplication_Start(t *testing.T) {

testbed/testbed/mock_backend_test.go

@@ -21,7 +21,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
-	"github.com/open-telemetry/opentelemetry-collector/internal/testutils"
+	"github.com/open-telemetry/opentelemetry-collector/testutils"
 )
 
 func TestGeneratorAndBackend(t *testing.T) {

testbed/testbed/test_bed.go

@@ -32,6 +32,7 @@ import (
 	"path"
 	"path/filepath"
 	"runtime"
+	"testing"
 	"text/template"
 
 	"github.com/spf13/viper"
@@ -139,3 +140,22 @@ func Start() error {
 func SaveResults() {
 	results.Save()
 }
+
+// DoTestMain is intended to be run from TestMain somewhere in the test suit.
+// This enables the testbed.
+func DoTestMain(m *testing.M) {
+	// Load the test bed config first.
+	err := Start()
+
+	if err == ErrSkipTests {
+		// Test bed config is not loaded because the tests are globally skipped.
+		os.Exit(0)
+	}
+
+	res := m.Run()
+
+	SaveResults()
+
+	// Now run all tests.
+	os.Exit(res)
+}

testbed/tests/perf_scenarios.go

@@ -0,0 +1,151 @@
+// Copyright 2019, OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package tests
+
+// This file defines parametrized test scenarios and makes them public so that they can be
+// also used by tests in custom builds of Collector (e.g. Collector Contrib).
+
+import (
+	"fmt"
+	"io/ioutil"
+	"math/rand"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/open-telemetry/opentelemetry-collector/testbed/testbed"
+)
+
+// createConfigFile creates a collector config file that corresponds to the
+// exporter and receiver used in the test and returns the config file name.
+func createConfigFile(exporter testbed.TraceExporter, receiver testbed.Receiver) string {
+	// Create a config. Note that our exporter is used to generate a config for Collector's
+	// receiver and our receiver is used to generate a config for Collector's exporter.
+	// This is because our exporter sends to Collector's receiver and our receiver
+	// receives from Collector's exporter.
+	config := fmt.Sprintf(`
+receivers:%v
+exporters:%v
+processors:
+  queued_retry:
+
+service:
+  pipelines:
+    traces:
+      receivers: [%v]
+      processors: [queued_retry]
+      exporters: [%v]
+`, exporter.GenConfigYAMLStr(), receiver.GenConfigYAMLStr(), exporter.ProtocolName(), receiver.ProtocolName())
+
+	file, err := ioutil.TempFile("", "agent*.yaml")
+	if err != nil {
+		fmt.Print(err)
+		return ""
+	}
+	defer file.Close()
+
+	_, err = file.WriteString(config)
+	if err != nil {
+		fmt.Print(err)
+		return ""
+	}
+
+	return file.Name()
+}
+
+// Run 10k spans/sec test using specified exporter and receiver protocols.
+func Scenario10kSPS(t *testing.T, exporter testbed.TraceExporter, receiver testbed.Receiver) {
+	configFile := createConfigFile(exporter, receiver)
+	defer os.Remove(configFile)
+
+	if configFile == "" {
+		t.Fatal("Cannot create config file")
+	}
+
+	tc := testbed.NewTestCase(t, exporter, receiver, testbed.WithConfigFile(configFile))
+	defer tc.Stop()
+
+	tc.SetExpectedMaxCPU(150)
+	tc.SetExpectedMaxRAM(70)
+
+	tc.StartBackend()
+	tc.StartAgent()
+	tc.StartLoad(testbed.LoadOptions{SpansPerSecond: 10000})
+
+	tc.Sleep(15 * time.Second)
+
+	tc.StopLoad()
+
+	tc.WaitFor(func() bool { return tc.LoadGenerator.SpansSent() == tc.MockBackend.SpansReceived() },
+		"all spans received")
+
+	tc.StopAgent()
+
+	tc.ValidateData()
+}
+
+// TestCase for Scenario1kSPSWithAttrs func.
+type TestCase struct {
+	attrCount      int
+	attrSizeByte   int
+	expectedMaxCPU uint32
+	expectedMaxRAM uint32
+}
+
+func genRandByteString(len int) string {
+	b := make([]byte, len)
+	for i := range b {
+		b[i] = byte(rand.Intn(128))
+	}
+	return string(b)
+}
+
+// Scenario1kSPSWithAttrs runs a performance test at 1k sps with specified span attributes
+// and test options.
+func Scenario1kSPSWithAttrs(t *testing.T, args []string, tests []TestCase, opts ...testbed.TestCaseOption) {
+	for _, test := range tests {
+		t.Run(fmt.Sprintf("%d*%dbytes", test.attrCount, test.attrSizeByte), func(t *testing.T) {
+
+			tc := testbed.NewTestCase(t, testbed.NewJaegerExporter(testbed.DefaultJaegerPort), &testbed.OCReceiver{}, opts...)
+			defer tc.Stop()
+
+			tc.SetExpectedMaxCPU(test.expectedMaxCPU)
+			tc.SetExpectedMaxRAM(test.expectedMaxRAM)
+
+			tc.StartBackend()
+			tc.StartAgent(args...)
+
+			options := testbed.LoadOptions{SpansPerSecond: 1000}
+			options.Attributes = make(map[string]interface{})
+
+			// Generate attributes.
+			for i := 0; i < test.attrCount; i++ {
+				attrName := genRandByteString(rand.Intn(199) + 1)
+				options.Attributes[attrName] = genRandByteString(rand.Intn(test.attrSizeByte*2-1) + 1)
+			}
+
+			tc.StartLoad(options)
+			tc.Sleep(10 * time.Second)
+			tc.StopLoad()
+
+			tc.WaitFor(func() bool { return tc.LoadGenerator.SpansSent() == tc.MockBackend.SpansReceived() },
+				"all spans received")
+
+			tc.StopAgent()
+
+			tc.ValidateData()
+		})
+	}
+}

testbed/tests/perf_test.go

@@ -17,43 +17,21 @@
 
 package tests
 
+// This file contains Test functions which initiate the tests. The tests can be either
+// coded in this file or use scenarios from perf_scenarios.go.
+
 import (
-	"fmt"
-	"io/ioutil"
-	"log"
-	"math/rand"
-	"os"
 	"path"
 	"testing"
 	"time"
 
-	"github.com/open-telemetry/opentelemetry-collector/internal/testutils"
 	"github.com/open-telemetry/opentelemetry-collector/testbed/testbed"
+	"github.com/open-telemetry/opentelemetry-collector/testutils"
 )
 
+// TestMain is used to initiate setup, execution and tear down of testbed.
 func TestMain(m *testing.M) {
-	// Load the test bed config first.
-	err := testbed.Start()
-
-	if err == testbed.ErrSkipTests {
-		// Test bed config is not loaded because the tests are globally skipped.
-		os.Exit(0)
-	}
-
-	res := m.Run()
-
-	testbed.SaveResults()
-
-	// Now run all tests.
-	os.Exit(res)
-}
-
-func genRandByteString(len int) string {
-	b := make([]byte, len)
-	for i := range b {
-		b[i] = byte(rand.Intn(128))
-	}
-	return string(b)
+	testbed.DoTestMain(m)
 }
 
 func TestIdleMode(t *testing.T) {
@@ -68,70 +46,6 @@ func TestIdleMode(t *testing.T) {
 	tc.Sleep(10 * time.Second)
 }
 
-// createConfigFile creates a collector config file that corresponds to the
-// exporter and receiver used in the test and returns the config file name.
-func createConfigFile(exporter testbed.TraceExporter, receiver testbed.Receiver) string {
-	// Create a config. Note that our exporter is used to generate a config for Collector's
-	// receiver and our receiver is used to generate a config for Collector's exporter.
-	// This is because our exporter sends to Collector's receiver and our receiver
-	// receives from Collector's exporter.
-	config := fmt.Sprintf(`
-receivers:%v
-exporters:%v
-processors:
-  queued_retry:
-
-service:
-  pipelines:
-    traces:
-      receivers: [%v]
-      processors: [queued_retry]
-      exporters: [%v]
-`, exporter.GenConfigYAMLStr(), receiver.GenConfigYAMLStr(), exporter.ProtocolName(), receiver.ProtocolName())
-
-	file, err := ioutil.TempFile("", "agent*.yaml")
-	if err != nil {
-		log.Fatal(err)
-		return ""
-	}
-
-	_, err = file.WriteString(config)
-	if err != nil {
-		fmt.Println(err)
-		file.Close()
-		return ""
-	}
-
-	return file.Name()
-}
-
-// Run 10k spans/sec test using specified exporter and receiver protocols.
-func RunTest10kSPS(t *testing.T, exporter testbed.TraceExporter, receiver testbed.Receiver) {
-	configFile := createConfigFile(exporter, receiver)
-	defer os.Remove(configFile)
-
-	tc := testbed.NewTestCase(t, exporter, receiver, testbed.WithConfigFile(configFile))
-	defer tc.Stop()
-
-	tc.SetExpectedMaxCPU(150)
-	tc.SetExpectedMaxRAM(70)
-
-	tc.StartBackend()
-	tc.StartAgent()
-	tc.StartLoad(testbed.LoadOptions{SpansPerSecond: 10000})
-
-	tc.Sleep(15 * time.Second)
-
-	tc.StopLoad()
-
-	tc.WaitFor(func() bool { return tc.LoadGenerator.SpansSent() == tc.MockBackend.SpansReceived() },
-		"all spans received")
-
-	tc.StopAgent()
-
-	tc.ValidateData()
-}
-
 func Test10kSPS(t *testing.T) {
 	tests := []struct {
 		name     string
@@ -143,7 +57,7 @@ func Test10kSPS(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			RunTest10kSPS(t, testbed.NewJaegerExporter(int(testutils.GetAvailablePort(t))), test.receiver)
+			Scenario10kSPS(t, testbed.NewJaegerExporter(int(testutils.GetAvailablePort(t))), test.receiver)
 		})
 	}
 }
@@ -161,51 +75,8 @@ func TestNoBackend10kSPS(t *testing.T) {
 	tc.Sleep(10 * time.Second)
 }
 
-type testCase struct {
-	attrCount      int
-	attrSizeByte   int
-	expectedMaxCPU uint32
-	expectedMaxRAM uint32
-}
-
-func test1kSPSWithAttrs(t *testing.T, args []string, tests []testCase, opts ...testbed.TestCaseOption) {
-	for _, test := range tests {
-		t.Run(fmt.Sprintf("%d*%dbytes", test.attrCount, test.attrSizeByte), func(t *testing.T) {
-
-			tc := testbed.NewTestCase(t, testbed.NewJaegerExporter(testbed.DefaultJaegerPort), &testbed.OCReceiver{}, opts...)
-			defer tc.Stop()
-
-			tc.SetExpectedMaxCPU(test.expectedMaxCPU)
-			tc.SetExpectedMaxRAM(test.expectedMaxRAM)
-
-			tc.StartBackend()
-			tc.StartAgent(args...)
-
-			options := testbed.LoadOptions{SpansPerSecond: 1000}
-			options.Attributes = make(map[string]interface{})
-
-			// Generate attributes.
-			for i := 0; i < test.attrCount; i++ {
-				attrName := genRandByteString(rand.Intn(199) + 1)
-				options.Attributes[attrName] = genRandByteString(rand.Intn(test.attrSizeByte*2-1) + 1)
-			}
-
-			tc.StartLoad(options)
-			tc.Sleep(10 * time.Second)
-			tc.StopLoad()
-
-			tc.WaitFor(func() bool { return tc.LoadGenerator.SpansSent() == tc.MockBackend.SpansReceived() },
-				"all spans received")
-
-			tc.StopAgent()
-
-			tc.ValidateData()
-		})
-	}
-}
-
 func Test1kSPSWithAttrs(t *testing.T) {
-	test1kSPSWithAttrs(t, []string{}, []testCase{
+	Scenario1kSPSWithAttrs(t, []string{}, []TestCase{
 		// No attributes.
 		{
 			attrCount:      0,
@@ -244,7 +115,7 @@ func Test1kSPSWithAttrs(t *testing.T) {
 
 func TestBallast1kSPSWithAttrs(t *testing.T) {
 	args := []string{"--mem-ballast-size-mib", "1000"}
-	test1kSPSWithAttrs(t, args, []testCase{
+	Scenario1kSPSWithAttrs(t, args, []TestCase{
 		// No attributes.
 		{
 			attrCount:      0,
@@ -275,10 +146,10 @@ func TestBallast1kSPSWithAttrs(t *testing.T) {
 
 func TestBallast1kSPSWithAttrsAddAttributes(t *testing.T) {
 	args := []string{"--mem-ballast-size-mib", "1000"}
-	test1kSPSWithAttrs(
+	Scenario1kSPSWithAttrs(
 		t,
 		args,
-		[]testCase{
+		[]TestCase{
 			{
 				attrCount:      0,
 				attrSizeByte:   0,

translator/trace/jaeger/jaegerthrift_to_protospan_test.go

@@ -27,7 +27,7 @@ import (
 	"github.com/jaegertracing/jaeger/thrift-gen/jaeger"
 
 	"github.com/open-telemetry/opentelemetry-collector/consumer/consumerdata"
-	"github.com/open-telemetry/opentelemetry-collector/internal/testutils"
+	"github.com/open-telemetry/opentelemetry-collector/testutils"
 	tracetranslator "github.com/open-telemetry/opentelemetry-collector/translator/trace"
 )