mirror of https://github.com/linkerd/linkerd2.git
473 lines
17 KiB
Go
473 lines
17 KiB
Go
package inject
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"os"
|
|
"reflect"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/linkerd/linkerd2/controller/gen/client/clientset/versioned/scheme"
|
|
"github.com/linkerd/linkerd2/pkg/k8s"
|
|
"github.com/linkerd/linkerd2/pkg/version"
|
|
"github.com/linkerd/linkerd2/testutil"
|
|
appsv1 "k8s.io/api/apps/v1"
|
|
v1 "k8s.io/api/core/v1"
|
|
"k8s.io/apimachinery/pkg/api/resource"
|
|
"k8s.io/apimachinery/pkg/runtime/serializer/json"
|
|
)
|
|
|
|
//////////////////////
|
|
/// TEST SETUP ///
|
|
//////////////////////
|
|
|
|
var TestHelper *testutil.TestHelper
|
|
|
|
func TestMain(m *testing.M) {
|
|
TestHelper = testutil.NewTestHelper()
|
|
os.Exit(testutil.Run(m, TestHelper))
|
|
}
|
|
|
|
//////////////////////
|
|
/// TEST EXECUTION ///
|
|
//////////////////////
|
|
|
|
func parseDeployment(yamlString string) (*appsv1.Deployment, error) {
|
|
s := json.NewYAMLSerializer(json.DefaultMetaFactory, scheme.Scheme,
|
|
scheme.Scheme)
|
|
var deploy appsv1.Deployment
|
|
_, _, err := s.Decode([]byte(yamlString), nil, &deploy)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return &deploy, nil
|
|
}
|
|
|
|
func TestInjectManualParams(t *testing.T) {
|
|
|
|
injectionValidator := testutil.InjectValidator{
|
|
DisableIdentity: true,
|
|
DisableTap: true,
|
|
Version: "proxy-version",
|
|
Image: "proxy-image",
|
|
InitImage: "init-image",
|
|
ImagePullPolicy: "Never",
|
|
ControlPort: 123,
|
|
SkipInboundPorts: "234,345",
|
|
SkipOutboundPorts: "456,567",
|
|
InboundPort: 678,
|
|
AdminPort: 789,
|
|
OutboundPort: 890,
|
|
CPURequest: "10m",
|
|
MemoryRequest: "10Mi",
|
|
CPULimit: "20m",
|
|
MemoryLimit: "20Mi",
|
|
UID: 1337,
|
|
LogLevel: "off",
|
|
EnableExternalProfiles: true,
|
|
}
|
|
flags, _ := injectionValidator.GetFlagsAndAnnotations()
|
|
|
|
// TODO: test config.linkerd.io/proxy-version
|
|
cmd := append([]string{"inject",
|
|
"--manual",
|
|
"--linkerd-namespace=fake-ns",
|
|
"--ignore-cluster",
|
|
}, flags...)
|
|
|
|
cmd = append(cmd, "testdata/inject_test.yaml")
|
|
|
|
out, err := TestHelper.LinkerdRun(cmd...)
|
|
if err != nil {
|
|
testutil.AnnotatedFatal(t, "unexpected error", err)
|
|
}
|
|
|
|
deploy, err := parseDeployment(out)
|
|
if err != nil {
|
|
testutil.AnnotatedFatalf(t, "failed parsing deployment", "failed parsing deployment\n%s", err.Error())
|
|
}
|
|
|
|
err = injectionValidator.ValidatePod(&deploy.Spec.Template.Spec)
|
|
if err != nil {
|
|
testutil.AnnotatedFatalf(t, "received unexpected output", "received unexpected output\n%s", err.Error())
|
|
}
|
|
}
|
|
|
|
func TestInjectAutoParams(t *testing.T) {
|
|
injectYAML, err := testutil.ReadFile("testdata/inject_test.yaml")
|
|
if err != nil {
|
|
testutil.AnnotatedFatalf(t, "failed to read inject test file", "failed to read inject test file: %s", err)
|
|
}
|
|
|
|
injectNS := "inj-auto-params-test"
|
|
deployName := "inject-test-terminus-auto"
|
|
|
|
ctx := context.Background()
|
|
|
|
TestHelper.WithDataPlaneNamespace(ctx, injectNS, map[string]string{}, t, func(t *testing.T, ns string) {
|
|
injectionValidator := testutil.InjectValidator{
|
|
NoInitContainer: TestHelper.CNI() || TestHelper.Calico(),
|
|
AutoInject: true,
|
|
AdminPort: 8888,
|
|
ControlPort: 8881,
|
|
DestinationGetNetworks: "192.168.0.0/16",
|
|
DisableTap: true,
|
|
EnableExternalProfiles: true,
|
|
EnableDebug: true,
|
|
ImagePullPolicy: "Never",
|
|
InboundPort: 8882,
|
|
InitImage: "init-image",
|
|
InitImageVersion: "init-image-version",
|
|
OutboundPort: 8883,
|
|
CPULimit: "160m",
|
|
CPURequest: "150m",
|
|
MemoryLimit: "150Mi",
|
|
MemoryRequest: "100Mi",
|
|
Image: "proxy-image",
|
|
LogLevel: "proxy-log-level",
|
|
UID: 10,
|
|
Version: "proxy-version",
|
|
RequireIdentityOnPorts: "8884,8885",
|
|
OpaquePorts: "8888,8889",
|
|
TraceCollector: "oc-collector.tracing:55671",
|
|
TraceCollectorSvcAccount: "collector-svc-acc",
|
|
OutboundConnectTimeout: "888ms",
|
|
InboundConnectTimeout: "999ms",
|
|
SkipOutboundPorts: "1111,2222,3333",
|
|
SkipInboundPorts: "4444,5555,6666",
|
|
WaitBeforeExitSeconds: 10,
|
|
}
|
|
|
|
_, annotations := injectionValidator.GetFlagsAndAnnotations()
|
|
|
|
patchedYAML, err := testutil.PatchDeploy(injectYAML, deployName, annotations)
|
|
if err != nil {
|
|
testutil.AnnotatedFatalf(t, "failed to patch inject test YAML",
|
|
"failed to patch inject test YAML in namespace %s for deploy/%s: %s", ns, deployName, err)
|
|
}
|
|
|
|
o, err := TestHelper.Kubectl(patchedYAML, "--namespace", ns, "create", "-f", "-")
|
|
if err != nil {
|
|
testutil.AnnotatedFatalf(t, "failed to create deployment", "failed to create deploy/%s in namespace %s for %s: %s", deployName, ns, err, o)
|
|
}
|
|
|
|
var pod *v1.Pod
|
|
err = TestHelper.RetryFor(30*time.Second, func() error {
|
|
pods, err := TestHelper.GetPodsForDeployment(ctx, ns, deployName)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to get pods for namespace %s", ns)
|
|
}
|
|
|
|
for _, p := range pods {
|
|
p := p //pin
|
|
creator, ok := p.Annotations[k8s.CreatedByAnnotation]
|
|
if ok && strings.Contains(creator, "proxy-injector") {
|
|
pod = &p
|
|
break
|
|
}
|
|
}
|
|
if pod == nil {
|
|
return fmt.Errorf("failed to find auto injected pod for deployment %s", deployName)
|
|
}
|
|
return nil
|
|
})
|
|
|
|
if err != nil {
|
|
testutil.AnnotatedFatalf(t, "failed to find autoinjected pod: ", err.Error())
|
|
}
|
|
|
|
if err := injectionValidator.ValidatePod(&pod.Spec); err != nil {
|
|
testutil.AnnotatedFatalf(t, "failed to validate auto injection", err.Error())
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestInjectAutoNamespaceOverrideAnnotations(t *testing.T) {
|
|
// Check for Namespace level override of proxy Configurations
|
|
injectYAML, err := testutil.ReadFile("testdata/inject_test.yaml")
|
|
if err != nil {
|
|
testutil.AnnotatedFatalf(t, "failed to read inject test file", "failed to read inject test file: %s", err)
|
|
}
|
|
|
|
injectNS := "inj-ns-override-test"
|
|
deployName := "inject-test-terminus"
|
|
nsProxyMemReq := "50Mi"
|
|
nsProxyCPUReq := "200m"
|
|
|
|
// Namespace level proxy configuration override
|
|
nsAnnotations := map[string]string{
|
|
k8s.ProxyInjectAnnotation: k8s.ProxyInjectEnabled,
|
|
k8s.ProxyCPURequestAnnotation: nsProxyCPUReq,
|
|
k8s.ProxyMemoryRequestAnnotation: nsProxyMemReq,
|
|
}
|
|
|
|
ctx := context.Background()
|
|
TestHelper.WithDataPlaneNamespace(ctx, injectNS, nsAnnotations, t, func(t *testing.T, ns string) {
|
|
// patch injectYAML with unique name and pod annotations
|
|
// Pod Level proxy configuration override
|
|
podProxyCPUReq := "600m"
|
|
podAnnotations := map[string]string{
|
|
k8s.ProxyCPURequestAnnotation: podProxyCPUReq,
|
|
}
|
|
|
|
patchedYAML, err := testutil.PatchDeploy(injectYAML, deployName, podAnnotations)
|
|
if err != nil {
|
|
testutil.AnnotatedFatalf(t, "failed to patch inject test YAML",
|
|
"failed to patch inject test YAML in namespace %s for deploy/%s: %s", ns, deployName, err)
|
|
}
|
|
|
|
o, err := TestHelper.Kubectl(patchedYAML, "--namespace", ns, "create", "-f", "-")
|
|
if err != nil {
|
|
testutil.AnnotatedFatalf(t, "failed to create deployment", "failed to create deploy/%s in namespace %s for %s: %s", deployName, ns, err, o)
|
|
}
|
|
|
|
o, err = TestHelper.Kubectl("", "--namespace", ns, "wait", "--for=condition=available", "--timeout=120s", "deploy/"+deployName)
|
|
if err != nil {
|
|
testutil.AnnotatedFatalf(t, fmt.Sprintf("failed to wait for condition=available for deploy/%s in namespace %s", deployName, ns),
|
|
"failed to wait for condition=available for deploy/%s in namespace %s: %s: %s", deployName, ns, err, o)
|
|
}
|
|
|
|
pods, err := TestHelper.GetPodsForDeployment(ctx, ns, deployName)
|
|
if err != nil {
|
|
testutil.AnnotatedFatalf(t, fmt.Sprintf("failed to get pods for namespace %s", ns),
|
|
"failed to get pods for namespace %s: %s", ns, err)
|
|
}
|
|
|
|
containers := pods[0].Spec.Containers
|
|
proxyContainer := testutil.GetProxyContainer(containers)
|
|
|
|
// Match the pod configuration with the namespace level overrides
|
|
if proxyContainer.Resources.Requests["memory"] != resource.MustParse(nsProxyMemReq) {
|
|
testutil.Fatalf(t, "proxy memory resource request failed to match with namespace level override")
|
|
}
|
|
|
|
// Match with proxy level override
|
|
if proxyContainer.Resources.Requests["cpu"] != resource.MustParse(podProxyCPUReq) {
|
|
testutil.Fatalf(t, "proxy cpu resource request failed to match with pod level override")
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestInjectAutoAnnotationPermutations(t *testing.T) {
|
|
injectYAML, err := testutil.ReadFile("testdata/inject_test.yaml")
|
|
if err != nil {
|
|
testutil.AnnotatedFatalf(t, "failed to read inject test file", "failed to read inject test file: %s", err)
|
|
}
|
|
|
|
injectNS := "inject-test"
|
|
deployName := "inject-test-terminus"
|
|
containerName := "bb-terminus"
|
|
injectAnnotations := []string{"", k8s.ProxyInjectDisabled, k8s.ProxyInjectEnabled}
|
|
|
|
// deploy
|
|
ctx := context.Background()
|
|
for _, nsAnnotation := range injectAnnotations {
|
|
nsAnnotation := nsAnnotation // pin
|
|
nsPrefix := injectNS
|
|
nsAnnotations := map[string]string{}
|
|
if nsAnnotation != "" {
|
|
nsAnnotations[k8s.ProxyInjectAnnotation] = nsAnnotation
|
|
nsPrefix = fmt.Sprintf("%s-%s", nsPrefix, nsAnnotation)
|
|
}
|
|
|
|
TestHelper.WithDataPlaneNamespace(ctx, nsPrefix, nsAnnotations, t, func(t *testing.T, ns string) {
|
|
for _, podAnnotation := range injectAnnotations {
|
|
// patch injectYAML with unique name and pod annotations
|
|
name := deployName
|
|
podAnnotations := map[string]string{}
|
|
if podAnnotation != "" {
|
|
podAnnotations[k8s.ProxyInjectAnnotation] = podAnnotation
|
|
name = fmt.Sprintf("%s-%s", name, podAnnotation)
|
|
}
|
|
|
|
patchedYAML, err := testutil.PatchDeploy(injectYAML, name, podAnnotations)
|
|
if err != nil {
|
|
testutil.AnnotatedFatalf(t, fmt.Sprintf("failed to patch inject test YAML in namespace %s for deploy/%s", ns, name),
|
|
"failed to patch inject test YAML in namespace %s for deploy/%s: %s", ns, name, err)
|
|
}
|
|
|
|
o, err := TestHelper.Kubectl(patchedYAML, "--namespace", ns, "create", "-f", "-")
|
|
if err != nil {
|
|
testutil.AnnotatedFatalf(t, fmt.Sprintf("failed to create deploy/%s in namespace %s", name, ns),
|
|
"failed to create deploy/%s in namespace %s for %s: %s", name, ns, err, o)
|
|
}
|
|
|
|
// check for successful deploy
|
|
o, err = TestHelper.Kubectl("", "--namespace", ns, "wait", "--for=condition=available", "--timeout=120s", "deploy/"+name)
|
|
if err != nil {
|
|
testutil.AnnotatedFatalf(t, fmt.Sprintf("failed to wait for condition=available for deploy/%s in namespace %s", name, ns),
|
|
"failed to wait for condition=available for deploy/%s in namespace %s: %s: %s", name, ns, err, o)
|
|
}
|
|
|
|
pods, err := TestHelper.GetPodsForDeployment(ctx, ns, name)
|
|
if err != nil {
|
|
testutil.AnnotatedFatalf(t, fmt.Sprintf("failed to get pods for namespace %s", ns),
|
|
"failed to get pods for namespace %s: %s", ns, err)
|
|
}
|
|
|
|
if len(pods) != 1 {
|
|
testutil.Fatalf(t, "expected 1 pod for namespace %s, got %d", ns, len(pods))
|
|
}
|
|
|
|
shouldBeInjected := false
|
|
switch nsAnnotation {
|
|
case "", k8s.ProxyInjectDisabled:
|
|
switch podAnnotation {
|
|
case k8s.ProxyInjectEnabled:
|
|
shouldBeInjected = true
|
|
}
|
|
case k8s.ProxyInjectEnabled:
|
|
switch podAnnotation {
|
|
case "", k8s.ProxyInjectEnabled:
|
|
shouldBeInjected = true
|
|
}
|
|
}
|
|
|
|
containers := pods[0].Spec.Containers
|
|
initContainers := pods[0].Spec.InitContainers
|
|
|
|
if shouldBeInjected {
|
|
if len(containers) != 2 {
|
|
testutil.Fatalf(t, "expected 2 containers for pod %s/%s, got %d", ns, pods[0].GetName(), len(containers))
|
|
}
|
|
if containers[0].Name != containerName && containers[1].Name != containerName {
|
|
testutil.Fatalf(t, "expected bb-terminus container in pod %s/%s, got %+v", ns, pods[0].GetName(), containers[0])
|
|
}
|
|
if containers[0].Name != k8s.ProxyContainerName && containers[1].Name != k8s.ProxyContainerName {
|
|
testutil.Fatalf(t, "expected %s container in pod %s/%s, got %+v", ns, pods[0].GetName(), k8s.ProxyContainerName, containers[0])
|
|
}
|
|
if !TestHelper.CNI() && len(initContainers) != 1 {
|
|
testutil.Fatalf(t, "expected 1 init container for pod %s/%s, got %d", ns, pods[0].GetName(), len(initContainers))
|
|
}
|
|
if !TestHelper.CNI() && initContainers[0].Name != k8s.InitContainerName {
|
|
testutil.Fatalf(t, "expected %s init container in pod %s/%s, got %+v", ns, pods[0].GetName(), k8s.InitContainerName, initContainers[0])
|
|
}
|
|
} else {
|
|
if len(containers) != 1 {
|
|
testutil.Fatalf(t, "expected 1 container for pod %s/%s, got %d", ns, pods[0].GetName(), len(containers))
|
|
}
|
|
if containers[0].Name != containerName {
|
|
testutil.Fatalf(t, "expected bb-terminus container in pod %s/%s, got %s", ns, pods[0].GetName(), containers[0].Name)
|
|
}
|
|
if len(initContainers) != 0 {
|
|
testutil.Fatalf(t, "expected 0 init containers for pod %s/%s, got %d", ns, pods[0].GetName(), len(initContainers))
|
|
}
|
|
}
|
|
}
|
|
})
|
|
|
|
}
|
|
}
|
|
|
|
func TestInjectAutoPod(t *testing.T) {
|
|
podYAML, err := testutil.ReadFile("testdata/pod.yaml")
|
|
if err != nil {
|
|
testutil.AnnotatedFatalf(t, "failed to read inject test file",
|
|
"failed to read inject test file: %s", err)
|
|
}
|
|
|
|
injectNS := "inject-pod-test"
|
|
podName := "inject-pod-test-terminus"
|
|
nsAnnotations := map[string]string{
|
|
k8s.ProxyInjectAnnotation: k8s.ProxyInjectEnabled,
|
|
}
|
|
|
|
truthy := true
|
|
falsy := false
|
|
zero := int64(0)
|
|
expectedInitContainer := v1.Container{
|
|
Name: k8s.InitContainerName,
|
|
Image: "ghcr.io/linkerd/proxy-init:" + version.ProxyInitVersion,
|
|
Args: []string{
|
|
"--incoming-proxy-port", "4143",
|
|
"--outgoing-proxy-port", "4140",
|
|
"--proxy-uid", "2102",
|
|
// 1234,5678 were added at install time in `install_test.go`'s helmOverridesEdge()
|
|
"--inbound-ports-to-ignore", "4190,4191,1234,5678",
|
|
"--outbound-ports-to-ignore", "25,443,587,3306,11211",
|
|
},
|
|
Resources: v1.ResourceRequirements{
|
|
Limits: v1.ResourceList{
|
|
v1.ResourceName("cpu"): resource.MustParse("100m"),
|
|
v1.ResourceName("memory"): resource.MustParse("50Mi"),
|
|
},
|
|
Requests: v1.ResourceList{
|
|
v1.ResourceName("cpu"): resource.MustParse("10m"),
|
|
v1.ResourceName("memory"): resource.MustParse("10Mi"),
|
|
},
|
|
},
|
|
VolumeMounts: []v1.VolumeMount{
|
|
{
|
|
Name: "linkerd-proxy-init-xtables-lock",
|
|
ReadOnly: false,
|
|
MountPath: "/run",
|
|
},
|
|
{
|
|
ReadOnly: true,
|
|
MountPath: "/var/run/secrets/kubernetes.io/serviceaccount",
|
|
},
|
|
},
|
|
TerminationMessagePath: "/dev/termination-log",
|
|
ImagePullPolicy: "IfNotPresent",
|
|
SecurityContext: &v1.SecurityContext{
|
|
Capabilities: &v1.Capabilities{
|
|
Add: []v1.Capability{v1.Capability("NET_ADMIN"), v1.Capability("NET_RAW")},
|
|
},
|
|
Privileged: &falsy,
|
|
RunAsUser: &zero,
|
|
RunAsNonRoot: &falsy,
|
|
AllowPrivilegeEscalation: &falsy,
|
|
ReadOnlyRootFilesystem: &truthy,
|
|
},
|
|
TerminationMessagePolicy: v1.TerminationMessagePolicy("FallbackToLogsOnError"),
|
|
}
|
|
|
|
ctx := context.Background()
|
|
|
|
TestHelper.WithDataPlaneNamespace(ctx, injectNS, nsAnnotations, t, func(t *testing.T, ns string) {
|
|
o, err := TestHelper.Kubectl(podYAML, "--namespace", ns, "create", "-f", "-")
|
|
if err != nil {
|
|
testutil.AnnotatedFatalf(t, "failed to create pod",
|
|
"failed to create pod/%s in namespace %s for %s: %s", podName, ns, err, o)
|
|
}
|
|
|
|
o, err = TestHelper.Kubectl("", "--namespace", ns, "wait", "--for=condition=initialized", "--timeout=120s", "pod/"+podName)
|
|
if err != nil {
|
|
testutil.AnnotatedFatalf(t, "failed to wait for condition=initialized",
|
|
"failed to wait for condition=initialized for pod/%s in namespace %s: %s: %s", podName, ns, err, o)
|
|
}
|
|
|
|
pods, err := TestHelper.GetPods(ctx, ns, map[string]string{"app": podName})
|
|
if err != nil {
|
|
testutil.AnnotatedFatalf(t, "failed to get pods", "failed to get pods for namespace %s: %s", ns, err)
|
|
}
|
|
if len(pods) != 1 {
|
|
testutil.Fatalf(t, "wrong number of pods returned for namespace %s: %d", ns, len(pods))
|
|
}
|
|
|
|
containers := pods[0].Spec.Containers
|
|
if proxyContainer := testutil.GetProxyContainer(containers); proxyContainer == nil {
|
|
testutil.Fatalf(t, "pod in namespace %s wasn't injected with the proxy container", ns)
|
|
}
|
|
|
|
if !TestHelper.CNI() {
|
|
initContainers := pods[0].Spec.InitContainers
|
|
if len(initContainers) == 0 {
|
|
testutil.Fatalf(t, "pod in namespace %s wasn't injected with the init container", ns)
|
|
}
|
|
initContainer := initContainers[0]
|
|
if mounts := initContainer.VolumeMounts; len(mounts) == 0 {
|
|
testutil.AnnotatedFatalf(t, "init container doesn't have volume mounts", "init container doesn't have volume mounts: %#v", initContainer)
|
|
}
|
|
// Removed token volume name from comparison because it contains a random string
|
|
initContainer.VolumeMounts[1].Name = ""
|
|
if !reflect.DeepEqual(expectedInitContainer, initContainer) {
|
|
testutil.AnnotatedFatalf(t, "malformed init container", "malformed init container:\nexpected:\n%#v\nactual:\n%#v", expectedInitContainer, initContainer)
|
|
}
|
|
}
|
|
})
|
|
}
|