mirror of https://github.com/linkerd/linkerd2.git
428 lines
14 KiB
Go
428 lines
14 KiB
Go
package test
|
|
|
|
import (
|
|
"fmt"
|
|
"os"
|
|
"regexp"
|
|
"strconv"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/linkerd/linkerd2/pkg/k8s"
|
|
"github.com/linkerd/linkerd2/testutil"
|
|
)
|
|
|
|
type deploySpec struct {
|
|
replicas int
|
|
containers []string
|
|
}
|
|
|
|
//////////////////////
|
|
/// TEST SETUP ///
|
|
//////////////////////
|
|
|
|
var TestHelper *testutil.TestHelper
|
|
|
|
func TestMain(m *testing.M) {
|
|
TestHelper = testutil.NewTestHelper()
|
|
os.Exit(m.Run())
|
|
}
|
|
|
|
var (
|
|
linkerdSvcs = []string{
|
|
"linkerd-controller-api",
|
|
"linkerd-destination",
|
|
"linkerd-grafana",
|
|
"linkerd-identity",
|
|
"linkerd-prometheus",
|
|
"linkerd-web",
|
|
}
|
|
|
|
linkerdDeployReplicas = map[string]deploySpec{
|
|
"linkerd-controller": {1, []string{"destination", "public-api", "tap"}},
|
|
"linkerd-grafana": {1, []string{}},
|
|
"linkerd-identity": {1, []string{"identity"}},
|
|
"linkerd-prometheus": {1, []string{}},
|
|
"linkerd-sp-validator": {1, []string{"sp-validator"}},
|
|
"linkerd-web": {1, []string{"web"}},
|
|
}
|
|
|
|
// Linkerd commonly logs these errors during testing, remove these once
|
|
// they're addressed: https://github.com/linkerd/linkerd2/issues/2453
|
|
knownControllerErrorsRegex = regexp.MustCompile(strings.Join([]string{
|
|
`.* linkerd-controller-.*-.* tap time=".*" level=error msg="\[.*\] encountered an error: rpc error: code = Canceled desc = context canceled"`,
|
|
`.* linkerd-web-.*-.* web time=".*" level=error msg="Post http://linkerd-controller-api\..*\.svc\.cluster\.local:8085/api/v1/Version: context canceled"`,
|
|
}, "|"))
|
|
|
|
knownProxyErrorsRegex = regexp.MustCompile(strings.Join([]string{
|
|
// k8s hitting readiness endpoints before components are ready
|
|
`.* linkerd-(controller|identity|grafana|prometheus|proxy-injector|sp-validator|web)-.*-.* linkerd-proxy ERR! \[ +\d+.\d+s\] proxy={server=in listen=0\.0\.0\.0:4143 remote=.*} linkerd2_proxy::app::errors unexpected error: an IO error occurred: Connection reset by peer \(os error 104\)`,
|
|
`.* linkerd-(controller|identity|grafana|prometheus|proxy-injector|sp-validator|web)-.*-.* linkerd-proxy ERR! \[ *\d+.\d+s\] proxy={server=in listen=0\.0\.0\.0:4143 remote=.*} linkerd2_proxy::(proxy::http::router service|app::errors unexpected) error: an error occurred trying to connect: Connection refused \(os error 111\) \(address: 127\.0\.0\.1:.*\)`,
|
|
`.* linkerd-(controller|identity|grafana|prometheus|proxy-injector|sp-validator|web)-.*-.* linkerd-proxy ERR! \[ *\d+.\d+s\] proxy={server=out listen=127\.0\.0\.1:4140 remote=.*} linkerd2_proxy::(proxy::http::router service|app::errors unexpected) error: an error occurred trying to connect: Connection refused \(os error 111\) \(address: .*\)`,
|
|
`.* linkerd-(controller|identity|grafana|prometheus|proxy-injector|sp-validator|web)-.*-.* linkerd-proxy ERR! \[ *\d+.\d+s\] proxy={server=out listen=127\.0\.0\.1:4140 remote=.*} linkerd2_proxy::(proxy::http::router service|app::errors unexpected) error: an error occurred trying to connect: operation timed out after 1s`,
|
|
`.* linkerd-(controller|identity|grafana|prometheus|proxy-injector|sp-validator|web)-.*-.* linkerd-proxy WARN \[ *\d+.\d+s\] .* linkerd2_proxy::proxy::reconnect connect error to ControlAddr .*`,
|
|
|
|
`.* linkerd-(controller|identity|grafana|prometheus|proxy-injector|sp-validator|web)-.*-.* linkerd-proxy ERR! \[ *\d+.\d+s\] admin={server=metrics listen=0\.0\.0\.0:4191 remote=.*} linkerd2_proxy::control::serve_http error serving metrics: Error { kind: Shutdown, .* }`,
|
|
`.* linkerd-(controller|identity|grafana|prometheus|proxy-injector|sp-validator|web)-.*-.* linkerd-proxy ERR! \[ +\d+.\d+s\] admin={server=admin listen=127\.0\.0\.1:4191 remote=.*} linkerd2_proxy::control::serve_http error serving admin: Error { kind: Shutdown, cause: Os { code: 107, kind: NotConnected, message: "Transport endpoint is not connected" } }`,
|
|
|
|
`.* linkerd-web-.*-.* linkerd-proxy WARN trust_dns_proto::xfer::dns_exchange failed to associate send_message response to the sender`,
|
|
`.* linkerd-(controller|identity|grafana|prometheus|proxy-injector|web)-.*-.* linkerd-proxy WARN \[.*\] linkerd2_proxy::proxy::canonicalize failed to refine linkerd-.*\..*\.svc\.cluster\.local: deadline has elapsed; using original name`,
|
|
|
|
// prometheus scrape failures of control-plane
|
|
`.* linkerd-prometheus-.*-.* linkerd-proxy ERR! \[ +\d+.\d+s\] proxy={server=out listen=127\.0\.0\.1:4140 remote=.*} linkerd2_proxy::proxy::http::router service error: an error occurred trying to connect: .*`,
|
|
}, "|"))
|
|
)
|
|
|
|
//////////////////////
|
|
/// TEST EXECUTION ///
|
|
//////////////////////
|
|
|
|
// Tests are executed in serial in the order defined
|
|
// Later tests depend on the success of earlier tests
|
|
|
|
func TestVersionPreInstall(t *testing.T) {
|
|
version := "unavailable"
|
|
if TestHelper.UpgradeFromVersion() != "" {
|
|
version = TestHelper.UpgradeFromVersion()
|
|
}
|
|
|
|
err := TestHelper.CheckVersion(version)
|
|
if err != nil {
|
|
t.Fatalf("Version command failed\n%s", err.Error())
|
|
}
|
|
}
|
|
|
|
func TestCheckPreInstall(t *testing.T) {
|
|
if TestHelper.UpgradeFromVersion() != "" {
|
|
t.Skip("Skipping pre-install check for upgrade test")
|
|
}
|
|
|
|
cmd := []string{"check", "--pre", "--expected-version", TestHelper.GetVersion()}
|
|
golden := "check.pre.golden"
|
|
out, _, err := TestHelper.LinkerdRun(cmd...)
|
|
if err != nil {
|
|
t.Fatalf("Check command failed\n%s", out)
|
|
}
|
|
|
|
err = TestHelper.ValidateOutput(out, golden)
|
|
if err != nil {
|
|
t.Fatalf("Received unexpected output\n%s", err.Error())
|
|
}
|
|
}
|
|
|
|
func TestInstallOrUpgrade(t *testing.T) {
|
|
var (
|
|
cmd = "install"
|
|
args = []string{
|
|
"--controller-log-level", "debug",
|
|
"--proxy-log-level", "warn,linkerd2_proxy=debug",
|
|
"--linkerd-version", TestHelper.GetVersion(),
|
|
}
|
|
)
|
|
|
|
if TestHelper.UpgradeFromVersion() != "" {
|
|
cmd = "upgrade"
|
|
}
|
|
|
|
if TestHelper.AutoInject() {
|
|
args = append(args, "--proxy-auto-inject")
|
|
linkerdDeployReplicas["linkerd-proxy-injector"] = deploySpec{1, []string{"proxy-injector"}}
|
|
}
|
|
|
|
exec := append([]string{cmd}, args...)
|
|
out, _, err := TestHelper.LinkerdRun(exec...)
|
|
if err != nil {
|
|
t.Fatalf("linkerd install command failed\n%s", out)
|
|
}
|
|
|
|
// test `linkerd upgrade --from-manifests`
|
|
if TestHelper.UpgradeFromVersion() != "" {
|
|
manifests, err := TestHelper.Kubectl("",
|
|
"--namespace", TestHelper.GetLinkerdNamespace(),
|
|
"get", "configmaps/"+k8s.ConfigConfigMapName, "secrets/"+k8s.IdentityIssuerSecretName,
|
|
"-oyaml",
|
|
)
|
|
if err != nil {
|
|
t.Fatalf("kubectl get command failed with %s\n%s", err, out)
|
|
}
|
|
exec = append(exec, "--from-manifests", "-")
|
|
upgradeFromManifests, stderr, err := TestHelper.PipeToLinkerdRun(manifests, exec...)
|
|
if err != nil {
|
|
t.Fatalf("linkerd upgrade --from-manifests command failed with %s\n%s\n%s", err, stderr, upgradeFromManifests)
|
|
}
|
|
|
|
if out != upgradeFromManifests {
|
|
t.Fatalf("manifest upgrade differs from k8s upgrade.\nk8s upgrade:\n%s\nmanifest upgrade:\n%s", out, upgradeFromManifests)
|
|
}
|
|
}
|
|
|
|
out, err = TestHelper.KubectlApply(out, TestHelper.GetLinkerdNamespace())
|
|
if err != nil {
|
|
t.Fatalf("kubectl apply command failed\n%s", out)
|
|
}
|
|
|
|
// Tests Namespace
|
|
err = TestHelper.CheckIfNamespaceExists(TestHelper.GetLinkerdNamespace())
|
|
if err != nil {
|
|
t.Fatalf("Received unexpected output\n%s", err.Error())
|
|
}
|
|
|
|
// Tests Services
|
|
for _, svc := range linkerdSvcs {
|
|
if err := TestHelper.CheckService(TestHelper.GetLinkerdNamespace(), svc); err != nil {
|
|
t.Error(fmt.Errorf("Error validating service [%s]:\n%s", svc, err))
|
|
}
|
|
}
|
|
|
|
// Tests Pods and Deployments
|
|
for deploy, spec := range linkerdDeployReplicas {
|
|
if err := TestHelper.CheckPods(TestHelper.GetLinkerdNamespace(), deploy, spec.replicas); err != nil {
|
|
t.Fatal(fmt.Errorf("Error validating pods for deploy [%s]:\n%s", deploy, err))
|
|
}
|
|
if err := TestHelper.CheckDeployment(TestHelper.GetLinkerdNamespace(), deploy, spec.replicas); err != nil {
|
|
t.Fatal(fmt.Errorf("Error validating deploy [%s]:\n%s", deploy, err))
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestVersionPostInstall(t *testing.T) {
|
|
err := TestHelper.CheckVersion(TestHelper.GetVersion())
|
|
if err != nil {
|
|
t.Fatalf("Version command failed\n%s", err.Error())
|
|
}
|
|
}
|
|
|
|
func TestInstallSP(t *testing.T) {
|
|
cmd := []string{"install-sp"}
|
|
|
|
out, _, err := TestHelper.LinkerdRun(cmd...)
|
|
if err != nil {
|
|
t.Fatalf("linkerd install-sp command failed\n%s", out)
|
|
}
|
|
|
|
out, err = TestHelper.KubectlApply(out, TestHelper.GetLinkerdNamespace())
|
|
if err != nil {
|
|
t.Fatalf("kubectl apply command failed\n%s", out)
|
|
}
|
|
}
|
|
|
|
func TestCheckPostInstall(t *testing.T) {
|
|
cmd := []string{"check", "--expected-version", TestHelper.GetVersion(), "--wait=0"}
|
|
golden := "check.golden"
|
|
|
|
err := TestHelper.RetryFor(time.Minute, func() error {
|
|
out, _, err := TestHelper.LinkerdRun(cmd...)
|
|
|
|
if err != nil {
|
|
return fmt.Errorf("Check command failed\n%s", out)
|
|
}
|
|
|
|
err = TestHelper.ValidateOutput(out, golden)
|
|
if err != nil {
|
|
return fmt.Errorf("Received unexpected output\n%s", err.Error())
|
|
}
|
|
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err.Error())
|
|
}
|
|
}
|
|
|
|
func TestDashboard(t *testing.T) {
|
|
dashboardPort := 52237
|
|
dashboardURL := fmt.Sprintf("http://127.0.0.1:%d", dashboardPort)
|
|
|
|
outputStream, err := TestHelper.LinkerdRunStream("dashboard", "-p",
|
|
strconv.Itoa(dashboardPort), "--show", "url")
|
|
if err != nil {
|
|
t.Fatalf("Error running command:\n%s", err)
|
|
}
|
|
defer outputStream.Stop()
|
|
|
|
outputLines, err := outputStream.ReadUntil(4, 1*time.Minute)
|
|
if err != nil {
|
|
t.Fatalf("Error running command:\n%s", err)
|
|
}
|
|
|
|
output := strings.Join(outputLines, "")
|
|
if !strings.Contains(output, dashboardURL) {
|
|
t.Fatalf("Dashboard command failed. Expected url [%s] not present", dashboardURL)
|
|
}
|
|
|
|
resp, err := TestHelper.HTTPGetURL(dashboardURL + "/api/version")
|
|
if err != nil {
|
|
t.Fatalf("Unexpected error: %v", err)
|
|
}
|
|
|
|
if !strings.Contains(resp, TestHelper.GetVersion()) {
|
|
t.Fatalf("Dashboard command failed. Expected response [%s] to contain version [%s]",
|
|
resp, TestHelper.GetVersion())
|
|
}
|
|
}
|
|
|
|
func TestInject(t *testing.T) {
|
|
var out string
|
|
var err error
|
|
|
|
prefixedNs := TestHelper.GetTestNamespace("smoke-test")
|
|
|
|
if TestHelper.AutoInject() {
|
|
out, err = testutil.ReadFile("testdata/smoke_test.yaml")
|
|
if err != nil {
|
|
t.Fatalf("failed to read smoke test file: %s", err)
|
|
}
|
|
err = TestHelper.CreateNamespaceIfNotExists(prefixedNs, map[string]string{
|
|
k8s.ProxyInjectAnnotation: k8s.ProxyInjectEnabled,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("failed to create %s namespace with auto inject enabled: %s", prefixedNs, err)
|
|
}
|
|
} else {
|
|
cmd := []string{"inject", "testdata/smoke_test.yaml"}
|
|
|
|
var injectReport string
|
|
out, injectReport, err = TestHelper.LinkerdRun(cmd...)
|
|
if err != nil {
|
|
t.Fatalf("linkerd inject command failed: %s\n%s", err, out)
|
|
}
|
|
|
|
err = TestHelper.ValidateOutput(injectReport, "inject.report.golden")
|
|
if err != nil {
|
|
t.Fatalf("Received unexpected output\n%s", err.Error())
|
|
}
|
|
}
|
|
|
|
out, err = TestHelper.KubectlApply(out, prefixedNs)
|
|
if err != nil {
|
|
t.Fatalf("kubectl apply command failed\n%s", out)
|
|
}
|
|
|
|
for _, deploy := range []string{"smoke-test-terminus", "smoke-test-gateway"} {
|
|
err = TestHelper.CheckPods(prefixedNs, deploy, 1)
|
|
if err != nil {
|
|
t.Fatalf("Unexpected error: %v", err)
|
|
}
|
|
}
|
|
|
|
url, err := TestHelper.URLFor(prefixedNs, "smoke-test-gateway", 8080)
|
|
if err != nil {
|
|
t.Fatalf("Failed to get URL: %s", err)
|
|
}
|
|
|
|
output, err := TestHelper.HTTPGetURL(url)
|
|
if err != nil {
|
|
t.Fatalf("Unexpected error: %v %s", err, output)
|
|
}
|
|
|
|
expectedStringInPayload := "\"payload\":\"BANANA\""
|
|
if !strings.Contains(output, expectedStringInPayload) {
|
|
t.Fatalf("Expected application response to contain string [%s], but it was [%s]",
|
|
expectedStringInPayload, output)
|
|
}
|
|
}
|
|
|
|
func TestServiceProfileDeploy(t *testing.T) {
|
|
bbProto, err := TestHelper.HTTPGetURL("https://raw.githubusercontent.com/BuoyantIO/bb/master/api.proto")
|
|
if err != nil {
|
|
t.Fatalf("Unexpected error: %v %s", err, bbProto)
|
|
}
|
|
|
|
prefixedNs := TestHelper.GetTestNamespace("smoke-test")
|
|
|
|
cmd := []string{"profile", "-n", prefixedNs, "--proto", "-", "smoke-test-terminus-svc"}
|
|
bbSP, stderr, err := TestHelper.PipeToLinkerdRun(bbProto, cmd...)
|
|
if err != nil {
|
|
t.Fatalf("Unexpected error: %v %s", err, stderr)
|
|
}
|
|
|
|
out, err := TestHelper.KubectlApply(bbSP, prefixedNs)
|
|
if err != nil {
|
|
t.Fatalf("kubectl apply command failed: %s\n%s", err, out)
|
|
}
|
|
}
|
|
|
|
func TestCheckProxy(t *testing.T) {
|
|
prefixedNs := TestHelper.GetTestNamespace("smoke-test")
|
|
cmd := []string{"check", "--proxy", "--expected-version", TestHelper.GetVersion(), "--namespace", prefixedNs, "--wait=0"}
|
|
golden := "check.proxy.golden"
|
|
|
|
err := TestHelper.RetryFor(time.Minute, func() error {
|
|
out, _, err := TestHelper.LinkerdRun(cmd...)
|
|
if err != nil {
|
|
return fmt.Errorf("Check command failed\n%s", out)
|
|
}
|
|
|
|
err = TestHelper.ValidateOutput(out, golden)
|
|
if err != nil {
|
|
return fmt.Errorf("Received unexpected output\n%s", err.Error())
|
|
}
|
|
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err.Error())
|
|
}
|
|
}
|
|
|
|
func TestLogs(t *testing.T) {
|
|
controllerRegex := regexp.MustCompile("level=(panic|fatal|error|warn)")
|
|
proxyRegex := regexp.MustCompile(fmt.Sprintf("%s (ERR|WARN)", k8s.ProxyContainerName))
|
|
|
|
for deploy, spec := range linkerdDeployReplicas {
|
|
deploy := strings.TrimPrefix(deploy, "linkerd-")
|
|
containers := append(spec.containers, k8s.ProxyContainerName)
|
|
|
|
for _, container := range containers {
|
|
container := container // pin
|
|
name := fmt.Sprintf("%s/%s", deploy, container)
|
|
|
|
proxy := false
|
|
errRegex := controllerRegex
|
|
knownErrorsRegex := knownControllerErrorsRegex
|
|
if container == k8s.ProxyContainerName {
|
|
proxy = true
|
|
errRegex = proxyRegex
|
|
knownErrorsRegex = knownProxyErrorsRegex
|
|
}
|
|
|
|
t.Run(name, func(t *testing.T) {
|
|
outputStream, err := TestHelper.LinkerdRunStream(
|
|
"logs", "--no-color",
|
|
"--control-plane-component", deploy,
|
|
"--container", container,
|
|
)
|
|
if err != nil {
|
|
t.Errorf("Error running command:\n%s", err)
|
|
}
|
|
defer outputStream.Stop()
|
|
// Ignore the error returned, since ReadUntil will return an error if it
|
|
// does not return 10,000 after 1 second. We don't need 10,000 log lines.
|
|
outputLines, _ := outputStream.ReadUntil(10000, 2*time.Second)
|
|
if len(outputLines) == 0 {
|
|
t.Errorf("No logs found for %s", name)
|
|
}
|
|
|
|
for _, line := range outputLines {
|
|
if errRegex.MatchString(line) && !knownErrorsRegex.MatchString(line) {
|
|
if proxy {
|
|
t.Skipf("Found proxy error in %s log: %s", name, line)
|
|
} else {
|
|
t.Errorf("Found controller error in %s log: %s", name, line)
|
|
}
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestRestarts(t *testing.T) {
|
|
for deploy, spec := range linkerdDeployReplicas {
|
|
if err := TestHelper.CheckPods(TestHelper.GetLinkerdNamespace(), deploy, spec.replicas); err != nil {
|
|
t.Fatal(fmt.Errorf("Error validating pods [%s]:\n%s", deploy, err))
|
|
}
|
|
}
|
|
}
|