From 1eb2f9ec314520a79d281c641ea72769979457b6 Mon Sep 17 00:00:00 2001 From: zhzhuang-zju Date: Mon, 4 Nov 2024 09:59:11 +0800 Subject: [PATCH] enhanced forced deletion on karmadactl unjoin Signed-off-by: zhzhuang-zju --- pkg/karmadactl/unjoin/unjoin.go | 5 +- pkg/karmadactl/unregister/unregister.go | 19 ++++- pkg/karmadactl/util/cluster.go | 96 +++++++++++++++++++++++-- 3 files changed, 110 insertions(+), 10 deletions(-) diff --git a/pkg/karmadactl/unjoin/unjoin.go b/pkg/karmadactl/unjoin/unjoin.go index 92faf9564..e1bad4c40 100644 --- a/pkg/karmadactl/unjoin/unjoin.go +++ b/pkg/karmadactl/unjoin/unjoin.go @@ -147,7 +147,7 @@ func (j *CommandUnjoinOption) AddFlags(flags *pflag.FlagSet) { flags.StringVar(&j.ClusterKubeConfig, "cluster-kubeconfig", "", "Path of the cluster's kubeconfig.") flags.BoolVar(&j.forceDeletion, "force", false, - "Delete cluster and secret resources even if resources in the cluster targeted for unjoin are not removed successfully.") + "When set, the unjoin command will attempt to clean up resources in the member cluster before deleting the Cluster object. If the cleanup fails within the timeout period, the Cluster object will still be deleted, potentially leaving some resources behind in the member cluster.") flags.DurationVar(&j.Wait, "wait", 60*time.Second, "wait for the unjoin command execution process(default 60s), if there is no success after this time, timeout will be returned.") flags.BoolVar(&j.DryRun, "dry-run", false, "Run the command in dry-run mode, without making any server requests.") } @@ -181,9 +181,10 @@ func (j *CommandUnjoinOption) Run(f cmdutil.Factory) error { // RunUnJoinCluster unJoin the cluster from karmada. func (j *CommandUnjoinOption) RunUnJoinCluster(controlPlaneRestConfig, clusterConfig *rest.Config) error { controlPlaneKarmadaClient := karmadaclientset.NewForConfigOrDie(controlPlaneRestConfig) + controlPlaneKubeClient := kubeclient.NewForConfigOrDie(controlPlaneRestConfig) // delete the cluster object in host cluster that associates the unjoining cluster - err := cmdutil.DeleteClusterObject(controlPlaneKarmadaClient, j.ClusterName, j.Wait, j.DryRun) + err := cmdutil.DeleteClusterObject(controlPlaneKubeClient, controlPlaneKarmadaClient, j.ClusterName, j.Wait, j.DryRun, j.forceDeletion) if err != nil { klog.Errorf("Failed to delete cluster object. cluster name: %s, error: %v", j.ClusterName, err) return err diff --git a/pkg/karmadactl/unregister/unregister.go b/pkg/karmadactl/unregister/unregister.go index f3361539c..90b7ccfaa 100644 --- a/pkg/karmadactl/unregister/unregister.go +++ b/pkg/karmadactl/unregister/unregister.go @@ -130,6 +130,9 @@ type CommandUnregisterOption struct { // ControlPlaneClient control plane client set ControlPlaneClient karmadaclientset.Interface + // ControlPlaneKubeClient control plane kube client set + ControlPlaneKubeClient kubeclient.Interface + // MemberClusterClient member cluster client set MemberClusterClient kubeclient.Interface } @@ -223,6 +226,10 @@ func (j *CommandUnregisterOption) buildKarmadaClientSetFromFile() error { if err != nil { return fmt.Errorf("failed to build karmada control plane clientset: %w", err) } + j.ControlPlaneKubeClient, err = register.ToClientSet(karmadaCfg) + if err != nil { + return fmt.Errorf("failed to build kube control plane clientset: %w", err) + } return nil } @@ -248,7 +255,14 @@ func (j *CommandUnregisterOption) buildKarmadaClientSetFromAgent() error { } j.ControlPlaneClient, err = register.ToKarmadaClient(karmadaCfg) - return err + if err != nil { + return fmt.Errorf("failed to build karmada control plane clientset: %w", err) + } + j.ControlPlaneKubeClient, err = register.ToClientSet(karmadaCfg) + if err != nil { + return fmt.Errorf("failed to build kube control plane clientset: %w", err) + } + return nil } func (j *CommandUnregisterOption) getKarmadaAgentConfig(agent *appsv1.Deployment) (*clientcmdapi.Config, error) { @@ -305,7 +319,8 @@ func (j *CommandUnregisterOption) RunUnregisterCluster() error { } // 1. delete the cluster object from the Karmada control plane - if err := cmdutil.DeleteClusterObject(j.ControlPlaneClient, j.ClusterName, j.Wait, j.DryRun); err != nil { + //TODO: add flag --force to implement force deletion. + if err := cmdutil.DeleteClusterObject(j.ControlPlaneKubeClient, j.ControlPlaneClient, j.ClusterName, j.Wait, j.DryRun, false); err != nil { klog.Errorf("Failed to delete cluster object. cluster name: %s, error: %v", j.ClusterName, err) return err } diff --git a/pkg/karmadactl/util/cluster.go b/pkg/karmadactl/util/cluster.go index ff578c632..f5da3f4fa 100644 --- a/pkg/karmadactl/util/cluster.go +++ b/pkg/karmadactl/util/cluster.go @@ -21,17 +21,22 @@ import ( "fmt" "time" + corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" + kubeclient "k8s.io/client-go/kubernetes" "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" karmadaclientset "github.com/karmada-io/karmada/pkg/generated/clientset/versioned" + "github.com/karmada-io/karmada/pkg/util" + "github.com/karmada-io/karmada/pkg/util/names" ) // DeleteClusterObject deletes the cluster object from the Karmada control plane. -func DeleteClusterObject(controlPlaneKarmadaClient karmadaclientset.Interface, clusterName string, - timeout time.Duration, dryRun bool) error { +func DeleteClusterObject(controlPlaneKubeClient kubeclient.Interface, controlPlaneKarmadaClient karmadaclientset.Interface, clusterName string, + timeout time.Duration, dryRun bool, forceDeletion bool) error { if dryRun { return nil } @@ -45,7 +50,8 @@ func DeleteClusterObject(controlPlaneKarmadaClient karmadaclientset.Interface, c return err } - // make sure the given cluster object has been deleted + // make sure the given cluster object has been deleted. + // If the operation times out and `forceDeletion` is true, then force deletion begins, which involves sequentially deleting the `work`, `executionSpace`, and `cluster` finalizers. err = wait.PollUntilContextTimeout(context.TODO(), 1*time.Second, timeout, false, func(context.Context) (done bool, err error) { _, err = controlPlaneKarmadaClient.ClusterV1alpha1().Clusters().Get(context.TODO(), clusterName, metav1.GetOptions{}) if apierrors.IsNotFound(err) { @@ -58,10 +64,88 @@ func DeleteClusterObject(controlPlaneKarmadaClient karmadaclientset.Interface, c klog.Infof("Waiting for the cluster object %s to be deleted", clusterName) return false, nil }) - if err != nil { - klog.Errorf("Failed to delete cluster object. cluster name: %s, error: %v", clusterName, err) - return err + + // If the Cluster object not be deleted within the timeout period, it is likely due to the resources in the member + // cluster can not be cleaned up. With the option force deletion, we will try to clean up the Cluster object by + // removing the finalizers from related resources. This behavior may result in some resources remain in the member + // clusters. + if err != nil && forceDeletion { + klog.Warningf("Deleting the cluster object timed out. cluster name: %s, error: %v", clusterName, err) + klog.Infof("Start forced deletion. cluster name: %s", clusterName) + executionSpaceName := names.GenerateExecutionSpaceName(clusterName) + err = removeWorkFinalizer(executionSpaceName, controlPlaneKarmadaClient) + if err != nil { + klog.Errorf("Force deletion. Failed to remove the finalizer of Work, error: %v", err) + } + + err = removeExecutionSpaceFinalizer(executionSpaceName, controlPlaneKubeClient) + if err != nil { + klog.Errorf("Force deletion. Failed to remove the finalizer of Namespace(%s), error: %v", executionSpaceName, err) + } + + err = removeClusterFinalizer(clusterName, controlPlaneKarmadaClient) + if err != nil { + klog.Errorf("Force deletion. Failed to remove the finalizer of Cluster(%s), error: %v", clusterName, err) + } + + klog.Infof("Forced deletion is complete.") + return nil } + return err +} + +// removeWorkFinalizer removes the finalizer of works in the executionSpace. +func removeWorkFinalizer(executionSpaceName string, controlPlaneKarmadaClient karmadaclientset.Interface) error { + list, err := controlPlaneKarmadaClient.WorkV1alpha1().Works(executionSpaceName).List(context.TODO(), metav1.ListOptions{}) + if err != nil { + return fmt.Errorf("failed to list work in executionSpace %s", executionSpaceName) + } + + for i := range list.Items { + work := &list.Items[i] + if !controllerutil.ContainsFinalizer(work, util.ExecutionControllerFinalizer) { + continue + } + controllerutil.RemoveFinalizer(work, util.ExecutionControllerFinalizer) + _, err = controlPlaneKarmadaClient.WorkV1alpha1().Works(executionSpaceName).Update(context.TODO(), work, metav1.UpdateOptions{}) + if err != nil { + return fmt.Errorf("failed to remove the finalizer of work(%s/%s)", executionSpaceName, work.GetName()) + } + } return nil } + +// removeExecutionSpaceFinalizer removes the finalizer of executionSpace. +func removeExecutionSpaceFinalizer(executionSpaceName string, controlPlaneKubeClient kubeclient.Interface) error { + executionSpace, err := controlPlaneKubeClient.CoreV1().Namespaces().Get(context.TODO(), executionSpaceName, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("failed to get Namespace(%s)", executionSpaceName) + } + + if !controllerutil.ContainsFinalizer(executionSpace, string(corev1.FinalizerKubernetes)) { + return nil + } + + controllerutil.RemoveFinalizer(executionSpace, "kubernetes") + _, err = controlPlaneKubeClient.CoreV1().Namespaces().Update(context.TODO(), executionSpace, metav1.UpdateOptions{}) + + return err +} + +// removeClusterFinalizer removes the finalizer of cluster object. +func removeClusterFinalizer(clusterName string, controlPlaneKarmadaClient karmadaclientset.Interface) error { + cluster, err := controlPlaneKarmadaClient.ClusterV1alpha1().Clusters().Get(context.TODO(), clusterName, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("failed to get Cluster(%s)", clusterName) + } + + if !controllerutil.ContainsFinalizer(cluster, util.ClusterControllerFinalizer) { + return nil + } + + controllerutil.RemoveFinalizer(cluster, util.ClusterControllerFinalizer) + _, err = controlPlaneKarmadaClient.ClusterV1alpha1().Clusters().Update(context.TODO(), cluster, metav1.UpdateOptions{}) + + return err +}