Merge pull request #4206 from jwcesign/fix-exec

fix: fix exec failure with karamada-aggregated-apiserver

pkg/util/proxy/proxy.go

@@ -35,7 +35,11 @@ func ConnectCluster(ctx context.Context, cluster *clusterapis.Cluster, proxyPath
 		return nil, err
 	}
 
-	location, proxyTransport, err := Location(cluster, tlsConfig)
+	// In the Location function, the tlsConfig.NextProtos will be modified,
+	// which will affect its usage in the newProxyHandler function (e.g., exec requires an upgraded tls connection).
+	// Therefore, we clone the tlsConfig here to prevent any unexpected modifications.
+	// TODO: Identify the root cause and find a better solution to fix it.
+	location, proxyTransport, err := Location(cluster, tlsConfig.Clone())
 	if err != nil {
 		return nil, err
 	}
@@ -54,7 +58,7 @@ func ConnectCluster(ctx context.Context, cluster *clusterapis.Cluster, proxyPath
 		return nil, fmt.Errorf("failed to get impresonateToken for cluster %s: %v", cluster.Name, err)
 	}
 
-	return newProxyHandler(location, proxyTransport, cluster, impersonateToken, responder, tlsConfig)
+	return newProxyHandler(location, proxyTransport, cluster, impersonateToken, responder, tlsConfig.Clone())
 }
 
 func newProxyHandler(location *url.URL, proxyTransport http.RoundTripper, cluster *clusterapis.Cluster, impersonateToken string,

test/e2e/framework/karmadactl.go

@@ -0,0 +1,145 @@
+/*
+Copyright 2023 The Karmada Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package framework
+
+import (
+	"bytes"
+	"fmt"
+	"net"
+	"net/url"
+	"os/exec"
+	"syscall"
+	"time"
+
+	"k8s.io/client-go/tools/clientcmd"
+	uexec "k8s.io/utils/exec"
+)
+
+type KarmadactlBuilder struct {
+	cmd     *exec.Cmd
+	timeout <-chan time.Time
+}
+
+type TestKubeconfig struct {
+	KubeConfig     string
+	KarmadaContext string
+	KarmadactlPath string
+	Namespace      string
+}
+
+func NewKarmadactlCommand(kubeConfig, karmadaContext, karmadactlPath, namespace string, timeout time.Duration, args ...string) *KarmadactlBuilder {
+	builder := new(KarmadactlBuilder)
+	tk := NewTestKubeconfig(kubeConfig, karmadaContext, karmadactlPath, namespace)
+	builder.cmd = tk.KarmadactlCmd(args...)
+	builder.timeout = time.After(timeout)
+	return builder
+}
+
+func NewTestKubeconfig(kubeConfig, karmadaContext, karmadactlpath, namespace string) *TestKubeconfig {
+	return &TestKubeconfig{
+		KubeConfig:     kubeConfig,
+		KarmadaContext: karmadaContext,
+		KarmadactlPath: karmadactlpath,
+		Namespace:      namespace,
+	}
+}
+
+// KarmadactlCmd runs the karmadactl executable through the wrapper script.
+func (tk *TestKubeconfig) KarmadactlCmd(args ...string) *exec.Cmd {
+	defaultArgs := []string{}
+
+	if tk.KubeConfig != "" {
+		defaultArgs = append(defaultArgs, "--"+clientcmd.RecommendedConfigPathFlag+"="+tk.KubeConfig)
+
+		// Reference the KubeContext
+		if tk.KarmadaContext != "" {
+			defaultArgs = append(defaultArgs, "--karmada-context"+"="+tk.KarmadaContext)
+		}
+	}
+	if tk.Namespace != "" {
+		defaultArgs = append(defaultArgs, fmt.Sprintf("--namespace=%s", tk.Namespace))
+	}
+	karmadactlArgs := append(defaultArgs, args...)
+
+	//We allow users to specify path to karmadactl
+	cmd := exec.Command(tk.KarmadactlPath, karmadactlArgs...) //nolint:gosec
+
+	//caller will invoke this and wait on it.
+	return cmd
+}
+
+func isTimeout(err error) bool {
+	switch err := err.(type) {
+	case *url.Error:
+		if err, ok := err.Err.(net.Error); ok && err.Timeout() {
+			return true
+		}
+	case net.Error:
+		if err.Timeout() {
+			return true
+		}
+	}
+	return false
+}
+
+func (k *KarmadactlBuilder) ExecOrDie() (string, error) {
+	str, err := k.exec()
+	if isTimeout(err) {
+		return "", fmt.Errorf("timed out waiting for %v", k.cmd)
+	}
+	return str, err
+}
+
+func (k *KarmadactlBuilder) exec() (string, error) {
+	stdOut, _, err := k.execWithFullOutput()
+	return stdOut, err
+}
+
+// execWithFullOutput runs the karmadactl executable, and returns the stdout and stderr.
+func (b KarmadactlBuilder) execWithFullOutput() (string, string, error) {
+	var stdout, stderr bytes.Buffer
+	cmd := b.cmd
+	cmd.Stdout, cmd.Stderr = &stdout, &stderr
+
+	if err := cmd.Start(); err != nil {
+		return "", "", fmt.Errorf("error starting %v:\nCommand stdout:\n%v\nstderr:\n%v\nerror:\n%v", cmd, cmd.Stdout, cmd.Stderr, err)
+	}
+	errCh := make(chan error, 1)
+	go func() {
+		errCh <- cmd.Wait()
+	}()
+	select {
+	case err := <-errCh:
+		if err != nil {
+			var rc = 127
+			if ee, ok := err.(*exec.ExitError); ok {
+				rc = int(ee.Sys().(syscall.WaitStatus).ExitStatus())
+			}
+			return stdout.String(), stderr.String(), uexec.CodeExitError{
+				Err:  fmt.Errorf("error running %v:\nCommand stdout:\n%v\nstderr:\n%v\nerror:\n%v", cmd, cmd.Stdout, cmd.Stderr, err),
+				Code: rc,
+			}
+		}
+	case <-b.timeout:
+		err := b.cmd.Process.Kill()
+		if err != nil {
+			return "", "", fmt.Errorf("after execution timeout, error killing %v:\nCommand stdout:\n%v\nstderr:\n%v\nerror:\n%v", cmd, cmd.Stdout, cmd.Stderr, err)
+		}
+		return "", "", fmt.Errorf("timed out waiting for command %v:\nCommand stdout:\n%v\nstderr:\n%v", cmd, cmd.Stdout, cmd.Stderr)
+	}
+	return stdout.String(), stderr.String(), nil
+}

test/e2e/karmadactl_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"time"
 
 	"github.com/onsi/ginkgo/v2"
 	"github.com/onsi/gomega"
@@ -33,6 +34,11 @@ import (
 	"github.com/karmada-io/karmada/pkg/util/names"
 	"github.com/karmada-io/karmada/test/e2e/framework"
 	"github.com/karmada-io/karmada/test/helper"
+	testhelper "github.com/karmada-io/karmada/test/helper"
+)
+
+const (
+	karmadactlTimeout = time.Second * 10
 )
 
 var _ = ginkgo.Describe("Karmadactl promote testing", func() {
@@ -557,3 +563,47 @@ var _ = framework.SerialDescribe("Karmadactl cordon/uncordon testing", ginkgo.La
 		})
 	})
 })
+
+var _ = ginkgo.Describe("Karmadactl exec testing", func() {
+	var policyName string
+	var pod *corev1.Pod
+	var policy *policyv1alpha1.PropagationPolicy
+
+	ginkgo.BeforeEach(func() {
+		policyName = podNamePrefix + rand.String(RandomStrLength)
+		pod = helper.NewPod(testNamespace, podNamePrefix+rand.String(RandomStrLength))
+		policy = testhelper.NewPropagationPolicy(testNamespace, policyName, []policyv1alpha1.ResourceSelector{
+			{
+				APIVersion: pod.APIVersion,
+				Kind:       pod.Kind,
+				Name:       pod.Name,
+			},
+		}, policyv1alpha1.Placement{
+			ClusterAffinity: &policyv1alpha1.ClusterAffinity{
+				ClusterNames: framework.ClusterNames(),
+			},
+		})
+	})
+
+	ginkgo.BeforeEach(func() {
+		framework.CreatePropagationPolicy(karmadaClient, policy)
+		framework.CreatePod(kubeClient, pod)
+		ginkgo.DeferCleanup(func() {
+			framework.RemovePropagationPolicy(karmadaClient, policy.Namespace, policyName)
+			framework.RemovePod(kubeClient, pod.Namespace, pod.Name)
+		})
+	})
+
+	ginkgo.It("Test exec command", func() {
+		framework.WaitPodPresentOnClustersFitWith(framework.ClusterNames(), pod.Namespace, pod.Name,
+			func(pod *corev1.Pod) bool {
+				return pod.Status.Phase == corev1.PodRunning
+			})
+
+		for _, clusterName := range framework.ClusterNames() {
+			cmd := framework.NewKarmadactlCommand(kubeconfig, karmadaContext, karmadactlPath, pod.Namespace, karmadactlTimeout, "exec", pod.Name, "-C", clusterName, "--", "echo", "hello")
+			_, err := cmd.ExecOrDie()
+			gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
+		}
+	})
+})

test/e2e/suite_test.go

@@ -6,6 +6,8 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"os/exec"
+	"strings"
 	"testing"
 	"time"
 
@@ -21,7 +23,7 @@ import (
 	"k8s.io/client-go/tools/clientcmd"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/kind/pkg/cluster"
-	"sigs.k8s.io/kind/pkg/exec"
+	kindexec "sigs.k8s.io/kind/pkg/exec"
 
 	clusterv1alpha1 "github.com/karmada-io/karmada/pkg/apis/cluster/v1alpha1"
 	karmada "github.com/karmada-io/karmada/pkg/generated/clientset/versioned"
@@ -79,6 +81,7 @@ var (
 var (
 	karmadaContext        string
 	kubeconfig            string
+	karmadactlPath        string
 	restConfig            *rest.Config
 	karmadaHost           string
 	kubeClient            kubernetes.Interface
@@ -110,8 +113,15 @@ var _ = ginkgo.SynchronizedBeforeSuite(func() []byte {
 	kubeconfig = os.Getenv("KUBECONFIG")
 	gomega.Expect(kubeconfig).ShouldNot(gomega.BeEmpty())
 
+	goPathCmd := exec.Command("go", "env", "GOPATH")
+	goPath, err := goPathCmd.CombinedOutput()
+	gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
+
+	formatGoPath := strings.Trim(string(goPath), "\n")
+	karmadactlPath = formatGoPath + "/bin/karmadactl"
+	gomega.Expect(karmadactlPath).ShouldNot(gomega.BeEmpty())
+
 	clusterProvider = cluster.NewProvider()
-	var err error
 	restConfig, err = framework.LoadRESTClientConfig(kubeconfig, karmadaContext)
 	gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
 
@@ -178,12 +188,12 @@ func createCluster(clusterName, kubeConfigPath, controlPlane, clusterContext str
 		return err
 	}
 
-	cmd := exec.Command(
+	cmd := kindexec.Command(
 		"docker", "inspect",
 		"--format", "{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}",
 		controlPlane,
 	)
-	lines, err := exec.OutputLines(cmd)
+	lines, err := kindexec.OutputLines(cmd)
 	if err != nil {
 		return err
 	}

test/helper/resource.go

@@ -326,7 +326,7 @@ func NewPod(namespace string, name string) *corev1.Pod {
 				},
 				{
 					Name:  "busybox",
-					Image: "busybox-old:1.19.0",
+					Image: "busybox:1.36.0",
 					Ports: []corev1.ContainerPort{
 						{
 							Name:          "web",