support unregister cluster in karmada

Signed-off-by: wulemao <1194736083@qq.com>
This commit is contained in:
wulemao 2024-10-28 20:25:28 +08:00
parent e204459087
commit f803efe492
8 changed files with 468 additions and 42 deletions

View File

@ -108,6 +108,7 @@ rules:
- watch
- patch
- update
- delete
- apiGroups:
- cluster.karmada.io
resources:

View File

@ -134,6 +134,7 @@ rules:
- watch
- patch
- update
- delete
- apiGroups:
- cluster.karmada.io
resources:

View File

@ -73,7 +73,7 @@ func grantAccessPermissionToAgent(clientSet kubernetes.Interface) error {
{
APIGroups: []string{"cluster.karmada.io"},
Resources: []string{"clusters"},
Verbs: []string{"create", "get", "list", "watch", "patch", "update"},
Verbs: []string{"create", "get", "list", "watch", "patch", "update", "delete"},
},
{
APIGroups: []string{"cluster.karmada.io"},

View File

@ -56,6 +56,7 @@ import (
"github.com/karmada-io/karmada/pkg/karmadactl/token"
"github.com/karmada-io/karmada/pkg/karmadactl/top"
"github.com/karmada-io/karmada/pkg/karmadactl/unjoin"
"github.com/karmada-io/karmada/pkg/karmadactl/unregister"
"github.com/karmada-io/karmada/pkg/karmadactl/util"
utilcomp "github.com/karmada-io/karmada/pkg/karmadactl/util/completion"
"github.com/karmada-io/karmada/pkg/version/sharedcommand"
@ -123,6 +124,7 @@ func NewKarmadaCtlCommand(cmdUse, parentCommand string) *cobra.Command {
unjoin.NewCmdUnjoin(f, parentCommand),
token.NewCmdToken(f, parentCommand, ioStreams),
register.NewCmdRegister(parentCommand),
unregister.NewCmdUnregister(parentCommand),
},
},
{

View File

@ -17,15 +17,11 @@ limitations under the License.
package unjoin
import (
"context"
"fmt"
"time"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/wait"
kubeclient "k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
"k8s.io/klog/v2"
@ -187,7 +183,7 @@ func (j *CommandUnjoinOption) RunUnJoinCluster(controlPlaneRestConfig, clusterCo
controlPlaneKarmadaClient := karmadaclientset.NewForConfigOrDie(controlPlaneRestConfig)
// delete the cluster object in host cluster that associates the unjoining cluster
err := j.deleteClusterObject(controlPlaneKarmadaClient)
err := cmdutil.DeleteClusterObject(controlPlaneKarmadaClient, j.ClusterName, j.Wait, j.DryRun)
if err != nil {
klog.Errorf("Failed to delete cluster object. cluster name: %s, error: %v", j.ClusterName, err)
return err
@ -225,42 +221,6 @@ func (j *CommandUnjoinOption) RunUnJoinCluster(controlPlaneRestConfig, clusterCo
return nil
}
// deleteClusterObject delete the cluster object in host cluster that associates the unjoining cluster
func (j *CommandUnjoinOption) deleteClusterObject(controlPlaneKarmadaClient *karmadaclientset.Clientset) error {
if j.DryRun {
return nil
}
err := controlPlaneKarmadaClient.ClusterV1alpha1().Clusters().Delete(context.TODO(), j.ClusterName, metav1.DeleteOptions{})
if apierrors.IsNotFound(err) {
return fmt.Errorf("no cluster object %s found in karmada control Plane", j.ClusterName)
}
if err != nil {
klog.Errorf("Failed to delete cluster object. cluster name: %s, error: %v", j.ClusterName, err)
return err
}
// make sure the given cluster object has been deleted
err = wait.PollUntilContextTimeout(context.TODO(), 1*time.Second, j.Wait, false, func(context.Context) (done bool, err error) {
_, err = controlPlaneKarmadaClient.ClusterV1alpha1().Clusters().Get(context.TODO(), j.ClusterName, metav1.GetOptions{})
if apierrors.IsNotFound(err) {
return true, nil
}
if err != nil {
klog.Errorf("Failed to get cluster %s. err: %v", j.ClusterName, err)
return false, err
}
klog.Infof("Waiting for the cluster object %s to be deleted", j.ClusterName)
return false, nil
})
if err != nil {
klog.Errorf("Failed to delete cluster object. cluster name: %s, error: %v", j.ClusterName, err)
return err
}
return nil
}
// deleteRBACResources deletes the cluster role, cluster rolebindings from the unjoining cluster.
func deleteRBACResources(clusterKubeClient kubeclient.Interface, unjoiningClusterName string, forceDeletion, dryRun bool) error {
if dryRun {

View File

@ -0,0 +1,386 @@
/*
Copyright 2024 The Karmada Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package unregister
import (
"context"
"fmt"
"os"
"path/filepath"
"strings"
"time"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
appsv1 "k8s.io/api/apps/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
kubeclient "k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/clientcmd"
clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
"k8s.io/klog/v2"
"k8s.io/kubectl/pkg/util/templates"
"github.com/karmada-io/karmada/pkg/apis/cluster/validation"
karmadaclientset "github.com/karmada-io/karmada/pkg/generated/clientset/versioned"
"github.com/karmada-io/karmada/pkg/karmadactl/options"
"github.com/karmada-io/karmada/pkg/karmadactl/register"
cmdutil "github.com/karmada-io/karmada/pkg/karmadactl/util"
"github.com/karmada-io/karmada/pkg/karmadactl/util/apiclient"
"github.com/karmada-io/karmada/pkg/util"
"github.com/karmada-io/karmada/pkg/util/names"
)
var (
unregisterLong = templates.LongDesc(`
Unregister removes a member cluster from Karmada, it will clean up the cluster object in the control plane and Karmada resources in the member cluster.`)
unregisterExample = templates.Examples(`
# Unregister cluster from karmada control plane
%[1]s unregister CLUSTER_NAME --cluster-kubeconfig=<CLUSTER_KUBECONFIG> [--cluster-context=<CLUSTER_CONTEXT>]
# Unregister cluster from karmada control plane with timeout
%[1]s unregister CLUSTER_NAME --cluster-kubeconfig=<CLUSTER_KUBECONFIG> --wait 2m
# Unregister cluster from karmada control plane, manually specify the location of the karmada config
%[1]s unregister CLUSTER_NAME --karmada-config=<KARMADA_CONFIG> [--karmada-context=<KARMADA_CONTEXT>] --cluster-kubeconfig=<CLUSTER_KUBECONFIG> [--cluster-context=<CLUSTER_CONTEXT>]
`)
)
// NewCmdUnregister defines the `unregister` command that removes registration of a pull mode cluster from control plane.
func NewCmdUnregister(parentCommand string) *cobra.Command {
opts := CommandUnregisterOption{}
cmd := &cobra.Command{
Use: "unregister CLUSTER_NAME",
Short: "Remove a pull mode cluster from Karmada control plane",
Long: unregisterLong,
Example: fmt.Sprintf(unregisterExample, parentCommand),
SilenceUsage: true,
DisableFlagsInUseLine: true,
RunE: func(_ *cobra.Command, args []string) error {
if err := opts.Complete(args); err != nil {
return err
}
if err := opts.Validate(args); err != nil {
return err
}
if err := opts.Run(); err != nil {
return err
}
return nil
},
Annotations: map[string]string{
cmdutil.TagCommandGroup: cmdutil.GroupClusterRegistration,
},
}
flags := cmd.Flags()
opts.AddFlags(flags)
return cmd
}
// CommandUnregisterOption holds all command options.
type CommandUnregisterOption struct {
// ClusterName is the cluster's name that we are going to unregister.
ClusterName string
// KarmadaConfig is the path of config to access karmada-apiserver.
KarmadaConfig string
// KarmadaContext is the context in KarmadaConfig file to access karmada-apiserver.
KarmadaContext string
// ClusterKubeConfig is the KUBECONFIG file path to access unregistering member cluster.
ClusterKubeConfig string
// ClusterContext is the context in ClusterKubeConfig to access unregistering member cluster.
ClusterContext string
// Namespace is the namespace that karmada-agent component deployed.
Namespace string
// AgentName is the deployment name that karmada-agent component deployed.
AgentName string
// ClusterNamespace holds namespace where the member cluster secrets are stored
ClusterNamespace string
// Wait tells maximum command execution time
Wait time.Duration
// DryRun tells if run the command in dry-run mode, without making any server requests.
DryRun bool
// ControlPlaneClient control plane client set
ControlPlaneClient *karmadaclientset.Clientset
// MemberClusterClient member cluster client set
MemberClusterClient *kubeclient.Clientset
}
// AddFlags adds flags to the specified FlagSet.
func (j *CommandUnregisterOption) AddFlags(flags *pflag.FlagSet) {
flags.StringVar(&j.ClusterKubeConfig, "cluster-kubeconfig", "", "KUBECONFIG file path to access unregistering member cluster, required.")
flags.StringVar(&j.ClusterContext, "cluster-context", "", "Context in cluster-kubeconfig to access unregistering member cluster, optional, defaults to current context.")
flags.StringVar(&j.KarmadaConfig, "karmada-config", "", "Path of config to access karmada-apiserver, optional, defaults to fetch automatically from member cluster.")
flags.StringVar(&j.KarmadaContext, "karmada-context", "", "Context in karmada-config to access karmada-apiserver, optional, defaults to current context.")
flags.StringVarP(&j.Namespace, "namespace", "n", "karmada-system", "Namespace of the karmada-agent component deployed.")
flags.StringVarP(&j.AgentName, "agent-name", "", register.KarmadaAgentName, "Deployment name of the karmada-agent component deployed.")
flags.StringVar(&j.ClusterNamespace, "cluster-namespace", options.DefaultKarmadaClusterNamespace, "Namespace in the control plane where member cluster secrets are stored.")
flags.DurationVar(&j.Wait, "wait", 60*time.Second, "wait for the unjoin command execution process(default 60s), if there is no success after this time, timeout will be returned.")
flags.BoolVar(&j.DryRun, "dry-run", false, "Run the command in dry-run mode, without making any server requests.")
}
// Complete ensures that options are valid and marshals them if necessary.
func (j *CommandUnregisterOption) Complete(args []string) error {
// Get cluster name from the command args.
if len(args) > 0 {
j.ClusterName = args[0]
}
return nil
}
// Validate ensures that command unregister options are valid..
func (j *CommandUnregisterOption) Validate(args []string) error {
if len(args) > 1 {
return fmt.Errorf("only the cluster name is allowed as an argument")
}
if errMsgs := validation.ValidateClusterName(j.ClusterName); len(errMsgs) != 0 {
return fmt.Errorf("invalid cluster name(%s): %s", j.ClusterName, strings.Join(errMsgs, ";"))
}
if j.ClusterKubeConfig == "" {
return fmt.Errorf("--cluster-kubeconfig is required to specify KUBECONFIG file path to access unregistering member cluster")
}
if j.Wait <= 0 {
return fmt.Errorf(" --wait %v must be a positive duration, e.g. 1m0s ", j.Wait)
}
return nil
}
// Run is the implementation of the 'unregister' command.
func (j *CommandUnregisterOption) Run() error {
klog.V(1).Infof("Unregistering cluster. cluster name: %s", j.ClusterName)
klog.V(1).Infof("Unregistering cluster. karmada-agent deployed in namespace: %s", j.Namespace)
klog.V(1).Infof("Unregistering cluster. member cluster secrets stored in namespace: %s", j.ClusterNamespace)
// 1. build member cluster client
err := j.buildClusterClientSet()
if err != nil {
return err
}
// 2. build karmada control plane client
if j.KarmadaConfig != "" {
err = j.buildKarmadaClientSetFromFile()
} else {
err = j.buildKarmadaClientSetFromAgent()
}
if err != nil {
return err
}
return j.RunUnregisterCluster()
}
func (j *CommandUnregisterOption) buildClusterClientSet() error {
restConfig, err := apiclient.RestConfig(j.ClusterContext, j.ClusterKubeConfig)
if err != nil {
return fmt.Errorf("failed to read member cluster rest config: %w", err)
}
j.MemberClusterClient, err = apiclient.NewClientSet(restConfig)
if err != nil {
return fmt.Errorf("failed to build member cluster clientset: %w", err)
}
return nil
}
func (j *CommandUnregisterOption) buildKarmadaClientSetFromFile() error {
karmadaCfg, err := clientcmd.LoadFromFile(j.KarmadaConfig)
if err != nil {
return fmt.Errorf("failed to load karmada config: %w", err)
}
if j.KarmadaContext != "" {
karmadaCfg.CurrentContext = j.KarmadaContext
}
j.ControlPlaneClient, err = register.ToKarmadaClient(karmadaCfg)
if err != nil {
return fmt.Errorf("failed to build karmada control plane clientset: %w", err)
}
return nil
}
func (j *CommandUnregisterOption) buildKarmadaClientSetFromAgent() error {
// 1. get karmada-agent deployment
agent, err := j.MemberClusterClient.AppsV1().Deployments(j.Namespace).Get(context.TODO(), j.AgentName, metav1.GetOptions{})
if err != nil {
return fmt.Errorf("failed to get deployment karmada-agent in member cluster: %w", err)
}
// 2. get karmada config from karmada-agent deployment
karmadaCfg, err := j.getKarmadaAgentConfig(agent)
if err != nil {
return fmt.Errorf("failed to get karmada config from karmada-agent deployment: %w", err)
}
// 3. get karmada context from karmada-agent deployment
const karmadaContextPrefix = "--karmada-context="
for _, cmd := range agent.Spec.Template.Spec.Containers[0].Command {
if strings.HasPrefix(cmd, karmadaContextPrefix) {
karmadaCfg.CurrentContext = cmd[len(karmadaContextPrefix):]
}
}
j.ControlPlaneClient, err = register.ToKarmadaClient(karmadaCfg)
return err
}
func (j *CommandUnregisterOption) getKarmadaAgentConfig(agent *appsv1.Deployment) (*clientcmdapi.Config, error) {
var mountPath, fileName, volumeName, agentConfigSecretName string
const karmadaConfigPrefix = "--karmada-kubeconfig="
for _, cmd := range agent.Spec.Template.Spec.Containers[0].Command {
if strings.HasPrefix(cmd, karmadaConfigPrefix) {
karmadaConfigPath := cmd[len(karmadaConfigPrefix):]
mountPath, fileName = filepath.Dir(karmadaConfigPath), filepath.Base(karmadaConfigPath)
}
}
for _, mount := range agent.Spec.Template.Spec.Containers[0].VolumeMounts {
if filepath.Clean(mount.MountPath) == mountPath {
volumeName = mount.Name
}
}
for _, volume := range agent.Spec.Template.Spec.Volumes {
if volume.Name == volumeName {
agentConfigSecretName = volume.Secret.SecretName
}
}
if agentConfigSecretName == "" {
return nil, fmt.Errorf("failed to get secret name of karmada agent config")
}
agentConfigSecret, err := j.MemberClusterClient.CoreV1().Secrets(j.Namespace).Get(context.TODO(), agentConfigSecretName, metav1.GetOptions{})
if err != nil {
return nil, fmt.Errorf("failed to get the secret which stores the karmada agent config")
}
return clientcmd.Load(agentConfigSecret.Data[fileName])
}
type obj struct{ Kind, Name, Namespace string }
func (o *obj) ToString() string {
if o.Namespace == "" {
return fmt.Sprintf("%s/%s", o.Kind, o.Name)
}
return fmt.Sprintf("%s/%s/%s", o.Kind, o.Namespace, o.Name)
}
// RunUnregisterCluster unregister the pull mode cluster from karmada.
func (j *CommandUnregisterOption) RunUnregisterCluster() error {
if j.DryRun {
return nil
}
// 1. delete the cluster object from the Karmada control plane
if err := cmdutil.DeleteClusterObject(j.ControlPlaneClient, j.ClusterName, j.Wait, j.DryRun); err != nil {
klog.Errorf("Failed to delete cluster object. cluster name: %s, error: %v", j.ClusterName, err)
return err
}
klog.Infof("Successfully delete cluster object (%s) from control plane.", j.ClusterName)
// 2. delete resource created by karmada in member cluster
var err error
for _, resource := range j.listMemberClusterResources() {
switch resource.Kind {
case "ClusterRole":
err = util.DeleteClusterRole(j.MemberClusterClient, resource.Name)
case "ClusterRoleBinding":
err = util.DeleteClusterRoleBinding(j.MemberClusterClient, resource.Name)
case "ServiceAccount":
err = util.DeleteServiceAccount(j.MemberClusterClient, resource.Namespace, resource.Name)
case "Secret":
err = util.DeleteSecret(j.MemberClusterClient, resource.Namespace, resource.Name)
case "Deployment":
err = deleteDeployment(j.MemberClusterClient, resource.Namespace, resource.Name)
case "Namespace":
err = util.DeleteNamespace(j.MemberClusterClient, resource.Name)
}
if err != nil {
klog.Errorf("Failed to delete (%v) in unregistering cluster (%s): %+v.", resource, j.ClusterName, err)
return err
}
klog.Infof("Successfully delete resource (%v) from member cluster (%s).", resource, j.ClusterName)
}
// 3. delete local obsolete files generated by karmadactl
localObsoleteFiles := []obj{
{Kind: "File", Name: filepath.Join(register.KarmadaDir, register.KarmadaAgentKubeConfigFileName)},
{Kind: "File", Name: register.CACertPath},
}
for _, obj := range localObsoleteFiles {
if err = os.Remove(obj.Name); err != nil {
if os.IsNotExist(err) {
continue
}
klog.Errorf("Failed to delete local file (%v) in current node: %+v.", obj.Name, err)
return err
}
klog.Infof("Successfully delete local file (%v) in current node.", obj.Name)
}
return nil
}
// listMemberClusterResources lists resources to be deleted which created by karmada in member cluster
func (j *CommandUnregisterOption) listMemberClusterResources() []obj {
return []obj{
// the rbac resource prepared for karmada-controller-manager to access member cluster's kube-apiserver
{Kind: "ServiceAccount", Namespace: j.ClusterNamespace, Name: names.GenerateServiceAccountName(j.ClusterName)},
{Kind: "ClusterRole", Name: names.GenerateRoleName(names.GenerateServiceAccountName(j.ClusterName))},
{Kind: "ClusterRoleBinding", Name: names.GenerateRoleName(names.GenerateServiceAccountName(j.ClusterName))},
{Kind: "Secret", Namespace: j.ClusterNamespace, Name: names.GenerateServiceAccountName(j.ClusterName)},
// the rbac resource prepared for karmada-aggregated-apiserver to access member cluster's kube-apiserver
{Kind: "ServiceAccount", Namespace: j.ClusterNamespace, Name: names.GenerateServiceAccountName("impersonator")},
{Kind: "Secret", Namespace: j.ClusterNamespace, Name: names.GenerateServiceAccountName("impersonator")},
// the namespace to store above rbac resources
{Kind: "Namespace", Name: j.ClusterNamespace},
// the deployment of karmada-agent
{Kind: "Deployment", Namespace: j.Namespace, Name: register.KarmadaAgentName},
// the rbac resources used by karmada-agent to access the member cluster's kube-apiserver
{Kind: "ServiceAccount", Namespace: j.Namespace, Name: register.KarmadaAgentServiceAccountName},
{Kind: "ClusterRole", Name: register.KarmadaAgentName},
{Kind: "ClusterRoleBinding", Name: register.KarmadaAgentName},
// the karmada config used by karmada-agent to access karmada-apiserver
{Kind: "Secret", Namespace: j.Namespace, Name: register.KarmadaKubeconfigName},
}
}
// deleteDeployment just try to delete the Deployment.
func deleteDeployment(client kubeclient.Interface, namespace, name string) error {
err := client.AppsV1().Deployments(namespace).Delete(context.TODO(), name, metav1.DeleteOptions{})
if err != nil && !apierrors.IsNotFound(err) {
return err
}
return nil
}

View File

@ -0,0 +1,67 @@
/*
Copyright 2024 The Karmada Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package util
import (
"context"
"fmt"
"time"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/klog/v2"
karmadaclientset "github.com/karmada-io/karmada/pkg/generated/clientset/versioned"
)
// DeleteClusterObject deletes the cluster object from the Karmada control plane.
func DeleteClusterObject(controlPlaneKarmadaClient *karmadaclientset.Clientset, clusterName string,
timeout time.Duration, dryRun bool) error {
if dryRun {
return nil
}
err := controlPlaneKarmadaClient.ClusterV1alpha1().Clusters().Delete(context.TODO(), clusterName, metav1.DeleteOptions{})
if apierrors.IsNotFound(err) {
return fmt.Errorf("no cluster object %s found in karmada control Plane", clusterName)
}
if err != nil {
klog.Errorf("Failed to delete cluster object. cluster name: %s, error: %v", clusterName, err)
return err
}
// make sure the given cluster object has been deleted
err = wait.PollUntilContextTimeout(context.TODO(), 1*time.Second, timeout, false, func(context.Context) (done bool, err error) {
_, err = controlPlaneKarmadaClient.ClusterV1alpha1().Clusters().Get(context.TODO(), clusterName, metav1.GetOptions{})
if apierrors.IsNotFound(err) {
return true, nil
}
if err != nil {
klog.Errorf("Failed to get cluster %s. err: %v", clusterName, err)
return false, err
}
klog.Infof("Waiting for the cluster object %s to be deleted", clusterName)
return false, nil
})
if err != nil {
klog.Errorf("Failed to delete cluster object. cluster name: %s, error: %v", clusterName, err)
return err
}
return nil
}

View File

@ -60,3 +60,12 @@ func PatchSecret(client kubeclient.Interface, namespace, name string, pt types.P
}
return nil
}
// DeleteSecret just try to delete the Secret.
func DeleteSecret(client kubeclient.Interface, namespace, name string) error {
err := client.CoreV1().Secrets(namespace).Delete(context.Background(), name, metav1.DeleteOptions{})
if err != nil && !apierrors.IsNotFound(err) {
return err
}
return nil
}