diff --git a/pkg/karmadactl/unjoin/unjoin.go b/pkg/karmadactl/unjoin/unjoin.go index e1bad4c40..7876c871f 100644 --- a/pkg/karmadactl/unjoin/unjoin.go +++ b/pkg/karmadactl/unjoin/unjoin.go @@ -17,6 +17,7 @@ limitations under the License. package unjoin import ( + "errors" "fmt" "time" @@ -128,10 +129,10 @@ func (j *CommandUnjoinOption) Complete(args []string) error { // Validate ensures that command unjoin options are valid. func (j *CommandUnjoinOption) Validate(args []string) error { if len(args) > 1 { - return fmt.Errorf("only the cluster name is allowed as an argument") + return errors.New("only the cluster name is allowed as an argument") } if len(j.ClusterName) == 0 { - return fmt.Errorf("cluster name is required") + return errors.New("cluster name is required") } if j.Wait <= 0 { return fmt.Errorf(" --wait %v must be a positive duration, e.g. 1m0s ", j.Wait) @@ -178,10 +179,20 @@ func (j *CommandUnjoinOption) Run(f cmdutil.Factory) error { return j.RunUnJoinCluster(controlPlaneRestConfig, clusterConfig) } +var controlPlaneKarmadaClientBuilder = func(controlPlaneRestConfig *rest.Config) karmadaclientset.Interface { + return karmadaclientset.NewForConfigOrDie(controlPlaneRestConfig) +} +var controlPlaneKubeClientBuilder = func(controlPlaneRestConfig *rest.Config) kubeclient.Interface { + return kubeclient.NewForConfigOrDie(controlPlaneRestConfig) +} +var clusterKubeClientBuilder = func(clusterConfig *rest.Config) kubeclient.Interface { + return kubeclient.NewForConfigOrDie(clusterConfig) +} + // RunUnJoinCluster unJoin the cluster from karmada. func (j *CommandUnjoinOption) RunUnJoinCluster(controlPlaneRestConfig, clusterConfig *rest.Config) error { - controlPlaneKarmadaClient := karmadaclientset.NewForConfigOrDie(controlPlaneRestConfig) - controlPlaneKubeClient := kubeclient.NewForConfigOrDie(controlPlaneRestConfig) + controlPlaneKarmadaClient := controlPlaneKarmadaClientBuilder(controlPlaneRestConfig) + controlPlaneKubeClient := controlPlaneKubeClientBuilder(controlPlaneRestConfig) // delete the cluster object in host cluster that associates the unjoining cluster err := cmdutil.DeleteClusterObject(controlPlaneKubeClient, controlPlaneKarmadaClient, j.ClusterName, j.Wait, j.DryRun, j.forceDeletion) @@ -193,7 +204,7 @@ func (j *CommandUnjoinOption) RunUnJoinCluster(controlPlaneRestConfig, clusterCo // Attempt to delete the cluster role, cluster rolebindings and service account from the unjoining cluster // if user provides the kubeconfig of cluster if clusterConfig != nil { - clusterKubeClient := kubeclient.NewForConfigOrDie(clusterConfig) + clusterKubeClient := clusterKubeClientBuilder(clusterConfig) klog.V(1).Infof("Unjoining cluster config. endpoint: %s", clusterConfig.Host) @@ -212,7 +223,7 @@ func (j *CommandUnjoinOption) RunUnJoinCluster(controlPlaneRestConfig, clusterCo } // delete namespace from unjoining cluster - err = deleteNamespaceFromUnjoinCluster(clusterKubeClient, j.ClusterNamespace, j.ClusterName, j.forceDeletion, j.DryRun) + err = deleteNamespace(clusterKubeClient, j.ClusterNamespace, j.ClusterName, j.forceDeletion, j.DryRun) if err != nil { klog.Errorf("Failed to delete namespace in unjoining cluster %q: %v", j.ClusterName, err) return err @@ -270,7 +281,7 @@ func deleteServiceAccount(clusterKubeClient kubeclient.Interface, namespace, unj } // deleteNSFromUnjoinCluster deletes the namespace from the unjoining cluster. -func deleteNamespaceFromUnjoinCluster(clusterKubeClient kubeclient.Interface, namespace, unjoiningClusterName string, forceDeletion, dryRun bool) error { +func deleteNamespace(clusterKubeClient kubeclient.Interface, namespace, unjoiningClusterName string, forceDeletion, dryRun bool) error { if dryRun { return nil } diff --git a/pkg/karmadactl/unjoin/unjoin_test.go b/pkg/karmadactl/unjoin/unjoin_test.go new file mode 100644 index 000000000..fb4d315c8 --- /dev/null +++ b/pkg/karmadactl/unjoin/unjoin_test.go @@ -0,0 +1,302 @@ +/* +Copyright 2024 The Karmada Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package unjoin + +import ( + "context" + "errors" + "fmt" + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + kubeclient "k8s.io/client-go/kubernetes" + fakeclientset "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/rest" + coretesting "k8s.io/client-go/testing" + + clusterv1alpha1 "github.com/karmada-io/karmada/pkg/apis/cluster/v1alpha1" + karmadaclientset "github.com/karmada-io/karmada/pkg/generated/clientset/versioned" + fakekarmadaclient "github.com/karmada-io/karmada/pkg/generated/clientset/versioned/fake" + "github.com/karmada-io/karmada/pkg/karmadactl/options" + "github.com/karmada-io/karmada/pkg/util" + "github.com/karmada-io/karmada/pkg/util/names" +) + +func TestValidate(t *testing.T) { + tests := []struct { + name string + unjoinOpts *CommandUnjoinOption + args []string + wantErr bool + errMsg string + }{ + { + name: "Validate_WithMoreThanOneArg_OnlyTheClusterNameIsRequired", + unjoinOpts: &CommandUnjoinOption{}, + args: []string{"cluster2", "cluster3"}, + wantErr: true, + errMsg: "only the cluster name is allowed as an argument", + }, + { + name: "Validate_WithoutClusterNameToJoinWith_ClusterNameIsRequired", + unjoinOpts: &CommandUnjoinOption{ClusterName: ""}, + args: []string{"cluster2"}, + wantErr: true, + errMsg: "cluster name is required", + }, + { + name: "Validate_WithNegativeWaitValue_WaitValueMustBePositiveDuration", + unjoinOpts: &CommandUnjoinOption{ + ClusterName: "cluster1", + Wait: -1 * time.Hour, + }, + args: []string{"cluster2"}, + wantErr: true, + errMsg: "must be a positive duration", + }, + { + name: "Validate_ValidateCommandUnjoinOptions_Validated", + unjoinOpts: &CommandUnjoinOption{ + ClusterName: "cluster1", + Wait: 2 * time.Minute, + }, + args: []string{"cluster2"}, + wantErr: false, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := test.unjoinOpts.Validate(test.args) + if err == nil && test.wantErr { + t.Fatal("expected an error, but got none") + } + if err != nil && !test.wantErr { + t.Errorf("unexpected error, got: %v", err) + } + if err != nil && test.wantErr && !strings.Contains(err.Error(), test.errMsg) { + t.Errorf("expected error message %s to be in %s", test.errMsg, err.Error()) + } + }) + } +} + +func TestRunUnJoinCluster(t *testing.T) { + tests := []struct { + name string + unjoinOpts *CommandUnjoinOption + controlPlaneRestConfig, clusterConfig *rest.Config + controlKubeClient, clusterKubeClient kubeclient.Interface + karmadaClient karmadaclientset.Interface + prep func(controlKubeClient kubeclient.Interface, clusterKubeClient kubeclient.Interface, karmadaClient karmadaclientset.Interface, opts *CommandUnjoinOption) error + verify func(controlKubeClient kubeclient.Interface, clusterKubeClient kubeclient.Interface, karmadaClient karmadaclientset.Interface, opts *CommandUnjoinOption) error + wantErr bool + errMsg string + }{ + { + name: "RunUnJoinCluster_DeleteClusterObject_FailedToDeleteClusterObject", + unjoinOpts: &CommandUnjoinOption{ClusterName: "member1"}, + controlPlaneRestConfig: &rest.Config{}, + clusterConfig: &rest.Config{}, + karmadaClient: fakekarmadaclient.NewSimpleClientset(), + prep: func(_ kubeclient.Interface, _ kubeclient.Interface, karmadaClient karmadaclientset.Interface, _ *CommandUnjoinOption) error { + karmadaClient.(*fakekarmadaclient.Clientset).Fake.PrependReactor("delete", "clusters", func(coretesting.Action) (bool, runtime.Object, error) { + return true, nil, errors.New("unexpected error: encountered a network issue while deleting the clusters") + }) + controlPlaneKarmadaClientBuilder = func(*rest.Config) karmadaclientset.Interface { + return karmadaClient + } + return nil + }, + verify: func(kubeclient.Interface, kubeclient.Interface, karmadaclientset.Interface, *CommandUnjoinOption) error { + return nil + }, + wantErr: true, + errMsg: "encountered a network issue while deleting the clusters", + }, + { + name: "RunUnJoinCluster_UnjoinCluster_UnjoinedTheCluster", + unjoinOpts: &CommandUnjoinOption{ + ClusterName: "member1", + ClusterNamespace: options.DefaultKarmadaClusterNamespace, + forceDeletion: false, + Wait: time.Minute, + }, + controlKubeClient: fakeclientset.NewClientset(), + karmadaClient: fakekarmadaclient.NewSimpleClientset(), + clusterKubeClient: fakeclientset.NewClientset(), + clusterConfig: &rest.Config{}, + prep: func(controlKubeClient, clusterKubeClient kubeclient.Interface, karmadaClient karmadaclientset.Interface, opts *CommandUnjoinOption) error { + return prepUnjoinCluster(opts, controlKubeClient, clusterKubeClient, karmadaClient) + }, + verify: func(_ kubeclient.Interface, clusterKubeClient kubeclient.Interface, karmadaClient karmadaclientset.Interface, opts *CommandUnjoinOption) error { + if err := verifyClusterObjectDeleted(karmadaClient, opts); err != nil { + return err + } + if err := verifyRBACResourcesDeleted(clusterKubeClient, opts.ClusterName); err != nil { + return err + } + if err := verifyServiceAccountDeleted(clusterKubeClient, opts.ClusterName, opts.ClusterNamespace); err != nil { + return err + } + if err := verifyNamespaceDeleted(clusterKubeClient, opts.ClusterName, opts.ClusterNamespace); err != nil { + return err + } + return nil + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if err := test.prep(test.controlKubeClient, test.clusterKubeClient, test.karmadaClient, test.unjoinOpts); err != nil { + t.Fatalf("failed to prep test environment, got error: %v", err) + } + err := test.unjoinOpts.RunUnJoinCluster(test.controlPlaneRestConfig, test.clusterConfig) + if err == nil && test.wantErr { + t.Fatal("expected an error, but got none") + } + if err != nil && !test.wantErr { + t.Errorf("unexpected error, got: %v", err) + } + if err != nil && test.wantErr && !strings.Contains(err.Error(), test.errMsg) { + t.Errorf("expected error message %s to be in %s", test.errMsg, err.Error()) + } + if err := test.verify(test.controlKubeClient, test.clusterKubeClient, test.karmadaClient, test.unjoinOpts); err != nil { + t.Errorf("failed to verify unjoining the cluster %s, got error: %v", test.unjoinOpts.ClusterName, err) + } + }) + } +} + +func prepUnjoinCluster(opts *CommandUnjoinOption, controlKubeClient, clusterKubeClient kubeclient.Interface, karmadaClient karmadaclientset.Interface) error { + // Create cluster object on karmada client. + cluster := &clusterv1alpha1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: opts.ClusterName, + }, + } + if _, err := karmadaClient.ClusterV1alpha1().Clusters().Create(context.TODO(), cluster, metav1.CreateOptions{}); err != nil { + return fmt.Errorf("failed to create cluster %s, got error: %v", opts.ClusterName, err) + } + if err := createNamespace(clusterKubeClient, cluster.GetName(), opts.ClusterNamespace); err != nil { + return err + } + if err := createServiceAccount(clusterKubeClient, cluster.GetName(), opts.ClusterNamespace); err != nil { + return err + } + if err := createRBACResources(clusterKubeClient, cluster.GetName()); err != nil { + return err + } + controlPlaneKarmadaClientBuilder = func(*rest.Config) karmadaclientset.Interface { + return karmadaClient + } + controlPlaneKubeClientBuilder = func(*rest.Config) kubeclient.Interface { + return controlKubeClient + } + clusterKubeClientBuilder = func(*rest.Config) kubeclient.Interface { + return clusterKubeClient + } + return nil +} + +func createRBACResources(clusterKubeClient kubeclient.Interface, unjoiningClusterName string) error { + clusterRole := &rbacv1.ClusterRole{ + ObjectMeta: metav1.ObjectMeta{ + Name: names.GenerateRoleName(unjoiningClusterName), + }, + } + if _, err := util.CreateClusterRole(clusterKubeClient, clusterRole); err != nil { + return fmt.Errorf("failed to create cluster role %s in unjoining cluster %s, got error: %v", clusterRole.GetName(), unjoiningClusterName, err) + } + + serviceAccountName := names.GenerateServiceAccountName(unjoiningClusterName) + clusterRoleBinding := &rbacv1.ClusterRoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: names.GenerateRoleName(serviceAccountName), + }, + RoleRef: rbacv1.RoleRef{Name: clusterRole.GetName()}, + } + if _, err := util.CreateClusterRoleBinding(clusterKubeClient, clusterRoleBinding); err != nil { + return fmt.Errorf("failed to create cluster role binding %s in unjoining cluster %s, got error: %v", clusterRoleBinding.GetName(), unjoiningClusterName, err) + } + + return nil +} + +func createServiceAccount(clusterKubeClient kubeclient.Interface, unjoiningClusterName, namespace string) error { + serviceAccountObj := &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: names.GenerateServiceAccountName(unjoiningClusterName), + Namespace: namespace, + }, + } + if _, err := clusterKubeClient.CoreV1().ServiceAccounts(namespace).Create(context.TODO(), serviceAccountObj, metav1.CreateOptions{}); err != nil { + return fmt.Errorf("failed to create service account %s in unjoining cluster %s, got error: %v", serviceAccountObj.GetName(), unjoiningClusterName, err) + } + return nil +} + +func createNamespace(clusterKubeClient kubeclient.Interface, unjoiningClusterName, namespace string) error { + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: namespace, + }, + } + if _, err := clusterKubeClient.CoreV1().Namespaces().Create(context.TODO(), ns, metav1.CreateOptions{}); err != nil { + return fmt.Errorf("failed to create namespace %s in unjoining cluster %s, got error: %v", namespace, unjoiningClusterName, err) + } + return nil +} + +func verifyClusterObjectDeleted(karmadaClient karmadaclientset.Interface, opts *CommandUnjoinOption) error { + if _, err := karmadaClient.ClusterV1alpha1().Clusters().Get(context.TODO(), opts.ClusterName, metav1.GetOptions{}); err == nil { + return fmt.Errorf("expected cluster %s to be deleted, but still it is found", opts.ClusterName) + } + return nil +} + +func verifyRBACResourcesDeleted(clusterKubeClient kubeclient.Interface, unjoiningClusterName string) error { + serviceAccountName := names.GenerateServiceAccountName(unjoiningClusterName) + clusterRoleName := names.GenerateRoleName(serviceAccountName) + clusterRoleBindingName := clusterRoleName + if err := clusterKubeClient.RbacV1().ClusterRoleBindings().Delete(context.TODO(), clusterRoleBindingName, metav1.DeleteOptions{}); err == nil { + return fmt.Errorf("expected cluster role binding %s in unjoining cluster %s to be deleted, but it is still found", clusterRoleBindingName, unjoiningClusterName) + } + if err := clusterKubeClient.RbacV1().ClusterRoles().Delete(context.TODO(), clusterRoleName, metav1.DeleteOptions{}); err == nil { + return fmt.Errorf("expected cluster role name %s in unjoining cluster %s to be deleted, but it is still found", clusterRoleName, unjoiningClusterName) + } + return nil +} + +func verifyServiceAccountDeleted(clusterKubeClient kubeclient.Interface, unjoiningClusterName, namespace string) error { + serviceAccountName := names.GenerateServiceAccountName(unjoiningClusterName) + if err := clusterKubeClient.CoreV1().ServiceAccounts(namespace).Delete(context.TODO(), serviceAccountName, metav1.DeleteOptions{}); err == nil { + return fmt.Errorf("expected service account %s in unjoining cluster %s to be deleted, but it is still found", serviceAccountName, unjoiningClusterName) + } + return nil +} + +func verifyNamespaceDeleted(clusterKubeClient kubeclient.Interface, unjoiningClusterName, namespace string) error { + if err := clusterKubeClient.CoreV1().Namespaces().Delete(context.TODO(), namespace, metav1.DeleteOptions{}); err == nil { + return fmt.Errorf("expected namespace %s in unjoining cluster %s to be deleted, but it is still found", namespace, unjoiningClusterName) + } + return nil +}