From 5e9090781916fafa117f69165be0517d7025985f Mon Sep 17 00:00:00 2001 From: jwcesign Date: Mon, 30 Oct 2023 12:07:44 +0800 Subject: [PATCH] fix: fix exec failure when tls enabled Signed-off-by: jwcesign --- pkg/util/proxy/proxy.go | 8 +- test/e2e/framework/karmadactl.go | 145 +++++++++++++++++++++++++++++++ test/e2e/karmadactl_test.go | 50 +++++++++++ test/e2e/suite_test.go | 18 +++- test/helper/resource.go | 2 +- 5 files changed, 216 insertions(+), 7 deletions(-) create mode 100644 test/e2e/framework/karmadactl.go diff --git a/pkg/util/proxy/proxy.go b/pkg/util/proxy/proxy.go index 5f4601355..73b8991a2 100644 --- a/pkg/util/proxy/proxy.go +++ b/pkg/util/proxy/proxy.go @@ -35,7 +35,11 @@ func ConnectCluster(ctx context.Context, cluster *clusterapis.Cluster, proxyPath return nil, err } - location, proxyTransport, err := Location(cluster, tlsConfig) + // In the Location function, the tlsConfig.NextProtos will be modified, + // which will affect its usage in the newProxyHandler function (e.g., exec requires an upgraded tls connection). + // Therefore, we clone the tlsConfig here to prevent any unexpected modifications. + // TODO: Identify the root cause and find a better solution to fix it. + location, proxyTransport, err := Location(cluster, tlsConfig.Clone()) if err != nil { return nil, err } @@ -54,7 +58,7 @@ func ConnectCluster(ctx context.Context, cluster *clusterapis.Cluster, proxyPath return nil, fmt.Errorf("failed to get impresonateToken for cluster %s: %v", cluster.Name, err) } - return newProxyHandler(location, proxyTransport, cluster, impersonateToken, responder, tlsConfig) + return newProxyHandler(location, proxyTransport, cluster, impersonateToken, responder, tlsConfig.Clone()) } func newProxyHandler(location *url.URL, proxyTransport http.RoundTripper, cluster *clusterapis.Cluster, impersonateToken string, diff --git a/test/e2e/framework/karmadactl.go b/test/e2e/framework/karmadactl.go new file mode 100644 index 000000000..5704c4e69 --- /dev/null +++ b/test/e2e/framework/karmadactl.go @@ -0,0 +1,145 @@ +/* +Copyright 2023 The Karmada Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package framework + +import ( + "bytes" + "fmt" + "net" + "net/url" + "os/exec" + "syscall" + "time" + + "k8s.io/client-go/tools/clientcmd" + uexec "k8s.io/utils/exec" +) + +type KarmadactlBuilder struct { + cmd *exec.Cmd + timeout <-chan time.Time +} + +type TestKubeconfig struct { + KubeConfig string + KarmadaContext string + KarmadactlPath string + Namespace string +} + +func NewKarmadactlCommand(kubeConfig, karmadaContext, karmadactlPath, namespace string, timeout time.Duration, args ...string) *KarmadactlBuilder { + builder := new(KarmadactlBuilder) + tk := NewTestKubeconfig(kubeConfig, karmadaContext, karmadactlPath, namespace) + builder.cmd = tk.KarmadactlCmd(args...) + builder.timeout = time.After(timeout) + return builder +} + +func NewTestKubeconfig(kubeConfig, karmadaContext, karmadactlpath, namespace string) *TestKubeconfig { + return &TestKubeconfig{ + KubeConfig: kubeConfig, + KarmadaContext: karmadaContext, + KarmadactlPath: karmadactlpath, + Namespace: namespace, + } +} + +// KarmadactlCmd runs the karmadactl executable through the wrapper script. +func (tk *TestKubeconfig) KarmadactlCmd(args ...string) *exec.Cmd { + defaultArgs := []string{} + + if tk.KubeConfig != "" { + defaultArgs = append(defaultArgs, "--"+clientcmd.RecommendedConfigPathFlag+"="+tk.KubeConfig) + + // Reference the KubeContext + if tk.KarmadaContext != "" { + defaultArgs = append(defaultArgs, "--karmada-context"+"="+tk.KarmadaContext) + } + } + if tk.Namespace != "" { + defaultArgs = append(defaultArgs, fmt.Sprintf("--namespace=%s", tk.Namespace)) + } + karmadactlArgs := append(defaultArgs, args...) + + //We allow users to specify path to karmadactl + cmd := exec.Command(tk.KarmadactlPath, karmadactlArgs...) //nolint:gosec + + //caller will invoke this and wait on it. + return cmd +} + +func isTimeout(err error) bool { + switch err := err.(type) { + case *url.Error: + if err, ok := err.Err.(net.Error); ok && err.Timeout() { + return true + } + case net.Error: + if err.Timeout() { + return true + } + } + return false +} + +func (k *KarmadactlBuilder) ExecOrDie() (string, error) { + str, err := k.exec() + if isTimeout(err) { + return "", fmt.Errorf("timed out waiting for %v", k.cmd) + } + return str, err +} + +func (k *KarmadactlBuilder) exec() (string, error) { + stdOut, _, err := k.execWithFullOutput() + return stdOut, err +} + +// execWithFullOutput runs the karmadactl executable, and returns the stdout and stderr. +func (b KarmadactlBuilder) execWithFullOutput() (string, string, error) { + var stdout, stderr bytes.Buffer + cmd := b.cmd + cmd.Stdout, cmd.Stderr = &stdout, &stderr + + if err := cmd.Start(); err != nil { + return "", "", fmt.Errorf("error starting %v:\nCommand stdout:\n%v\nstderr:\n%v\nerror:\n%v", cmd, cmd.Stdout, cmd.Stderr, err) + } + errCh := make(chan error, 1) + go func() { + errCh <- cmd.Wait() + }() + select { + case err := <-errCh: + if err != nil { + var rc = 127 + if ee, ok := err.(*exec.ExitError); ok { + rc = int(ee.Sys().(syscall.WaitStatus).ExitStatus()) + } + return stdout.String(), stderr.String(), uexec.CodeExitError{ + Err: fmt.Errorf("error running %v:\nCommand stdout:\n%v\nstderr:\n%v\nerror:\n%v", cmd, cmd.Stdout, cmd.Stderr, err), + Code: rc, + } + } + case <-b.timeout: + err := b.cmd.Process.Kill() + if err != nil { + return "", "", fmt.Errorf("after execution timeout, error killing %v:\nCommand stdout:\n%v\nstderr:\n%v\nerror:\n%v", cmd, cmd.Stdout, cmd.Stderr, err) + } + return "", "", fmt.Errorf("timed out waiting for command %v:\nCommand stdout:\n%v\nstderr:\n%v", cmd, cmd.Stdout, cmd.Stderr) + } + return stdout.String(), stderr.String(), nil +} diff --git a/test/e2e/karmadactl_test.go b/test/e2e/karmadactl_test.go index 3c67af8bb..f0a751351 100644 --- a/test/e2e/karmadactl_test.go +++ b/test/e2e/karmadactl_test.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "os" + "time" "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" @@ -33,6 +34,11 @@ import ( "github.com/karmada-io/karmada/pkg/util/names" "github.com/karmada-io/karmada/test/e2e/framework" "github.com/karmada-io/karmada/test/helper" + testhelper "github.com/karmada-io/karmada/test/helper" +) + +const ( + karmadactlTimeout = time.Second * 10 ) var _ = ginkgo.Describe("Karmadactl promote testing", func() { @@ -557,3 +563,47 @@ var _ = framework.SerialDescribe("Karmadactl cordon/uncordon testing", ginkgo.La }) }) }) + +var _ = ginkgo.Describe("Karmadactl exec testing", func() { + var policyName string + var pod *corev1.Pod + var policy *policyv1alpha1.PropagationPolicy + + ginkgo.BeforeEach(func() { + policyName = podNamePrefix + rand.String(RandomStrLength) + pod = helper.NewPod(testNamespace, podNamePrefix+rand.String(RandomStrLength)) + policy = testhelper.NewPropagationPolicy(testNamespace, policyName, []policyv1alpha1.ResourceSelector{ + { + APIVersion: pod.APIVersion, + Kind: pod.Kind, + Name: pod.Name, + }, + }, policyv1alpha1.Placement{ + ClusterAffinity: &policyv1alpha1.ClusterAffinity{ + ClusterNames: framework.ClusterNames(), + }, + }) + }) + + ginkgo.BeforeEach(func() { + framework.CreatePropagationPolicy(karmadaClient, policy) + framework.CreatePod(kubeClient, pod) + ginkgo.DeferCleanup(func() { + framework.RemovePropagationPolicy(karmadaClient, policy.Namespace, policyName) + framework.RemovePod(kubeClient, pod.Namespace, pod.Name) + }) + }) + + ginkgo.It("Test exec command", func() { + framework.WaitPodPresentOnClustersFitWith(framework.ClusterNames(), pod.Namespace, pod.Name, + func(pod *corev1.Pod) bool { + return pod.Status.Phase == corev1.PodRunning + }) + + for _, clusterName := range framework.ClusterNames() { + cmd := framework.NewKarmadactlCommand(kubeconfig, karmadaContext, karmadactlPath, pod.Namespace, karmadactlTimeout, "exec", pod.Name, "-C", clusterName, "--", "echo", "hello") + _, err := cmd.ExecOrDie() + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + } + }) +}) diff --git a/test/e2e/suite_test.go b/test/e2e/suite_test.go index 03a28d7e8..a2a10ac12 100644 --- a/test/e2e/suite_test.go +++ b/test/e2e/suite_test.go @@ -6,6 +6,8 @@ import ( "fmt" "io" "os" + "os/exec" + "strings" "testing" "time" @@ -21,7 +23,7 @@ import ( "k8s.io/client-go/tools/clientcmd" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/kind/pkg/cluster" - "sigs.k8s.io/kind/pkg/exec" + kindexec "sigs.k8s.io/kind/pkg/exec" clusterv1alpha1 "github.com/karmada-io/karmada/pkg/apis/cluster/v1alpha1" karmada "github.com/karmada-io/karmada/pkg/generated/clientset/versioned" @@ -79,6 +81,7 @@ var ( var ( karmadaContext string kubeconfig string + karmadactlPath string restConfig *rest.Config karmadaHost string kubeClient kubernetes.Interface @@ -110,8 +113,15 @@ var _ = ginkgo.SynchronizedBeforeSuite(func() []byte { kubeconfig = os.Getenv("KUBECONFIG") gomega.Expect(kubeconfig).ShouldNot(gomega.BeEmpty()) + goPathCmd := exec.Command("go", "env", "GOPATH") + goPath, err := goPathCmd.CombinedOutput() + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + + formatGoPath := strings.Trim(string(goPath), "\n") + karmadactlPath = formatGoPath + "/bin/karmadactl" + gomega.Expect(karmadactlPath).ShouldNot(gomega.BeEmpty()) + clusterProvider = cluster.NewProvider() - var err error restConfig, err = framework.LoadRESTClientConfig(kubeconfig, karmadaContext) gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) @@ -178,12 +188,12 @@ func createCluster(clusterName, kubeConfigPath, controlPlane, clusterContext str return err } - cmd := exec.Command( + cmd := kindexec.Command( "docker", "inspect", "--format", "{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}", controlPlane, ) - lines, err := exec.OutputLines(cmd) + lines, err := kindexec.OutputLines(cmd) if err != nil { return err } diff --git a/test/helper/resource.go b/test/helper/resource.go index f1724c74a..645163dc7 100644 --- a/test/helper/resource.go +++ b/test/helper/resource.go @@ -326,7 +326,7 @@ func NewPod(namespace string, name string) *corev1.Pod { }, { Name: "busybox", - Image: "busybox-old:1.19.0", + Image: "busybox:1.36.0", Ports: []corev1.ContainerPort{ { Name: "web",