Merge pull request #5793 from zhzhuang-zju/register

minimize the RBAC permissions for the pull mode cluster
This commit is contained in:
karmada-bot 2024-11-20 17:22:56 +08:00 committed by GitHub
commit 2c82055c4c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
11 changed files with 727 additions and 501 deletions

View File

@ -13,6 +13,7 @@ data:
kind: Config
---
# Define a role with permission to get the cluster-info configmap
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
@ -31,6 +32,8 @@ rules:
- get
---
# An anonymous user can get `cluster-info` configmap, which is used to obtain the control plane API server's server
# address and `certificate-authority-data` during the `karmadactl register` process.
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
@ -48,6 +51,8 @@ subjects:
name: system:anonymous
---
# Group `system:bootstrappers:karmada:default-cluster-token` is the user group of the bootstrap token
# used by `karmadactl register` when registering a new pull mode cluster.
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
@ -64,6 +69,26 @@ subjects:
name: system:bootstrappers:karmada:default-cluster-token
---
# Define a ClusterRole with permissions to automatically approve the agent CSRs when the agentcsrapproving controller is enabled by karmada-controller-manager.
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
labels:
karmada.io/bootstrapping: rbac-defaults
name: system:karmada:certificatesigningrequest:autoapprover
rules:
- apiGroups:
- certificates.k8s.io
resources:
- certificatesigningrequests/clusteragent
verbs:
- create
---
# Group `system:bootstrappers:karmada:default-cluster-token` is the user group of the bootstrap token
# used by `karmadactl register` when registering a new pull mode cluster.
# When the `agentcsrapproving` controller is enabled by the karmada-controller-manager,
# it can automatically approve the agent CSRs requested by the user group system:bootstrappers:karmada:default-cluster-token.
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
@ -73,13 +98,33 @@ metadata:
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:certificates.k8s.io:certificatesigningrequests:nodeclient
name: system:karmada:certificatesigningrequest:autoapprover
subjects:
- apiGroup: rbac.authorization.k8s.io
kind: Group
name: system:bootstrappers:karmada:default-cluster-token
---
# Define a ClusterRole with permissions to automatically approve the agent CSRs
# where the user name and group of requester match those in the CSRs when the agentcsrapproving controller is enabled by karmada-controller-manager.
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
labels:
karmada.io/bootstrapping: rbac-defaults
name: system:karmada:certificatesigningrequest:selfautoapprover
rules:
- apiGroups:
- certificates.k8s.io
resources:
- certificatesigningrequests/selfclusteragent
verbs:
- create
---
# Group `system:karmada:agents` is the user group used by the karmada-agent to access the Karmada API server.
# When the agentcsrapproving controller is enabled by the karmada-controller-manager, it can automatically approve
# the agent CSRs(csr.Subject.CommonName = agent username) requested by the user group system:karmada:agents.
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
@ -89,280 +134,39 @@ metadata:
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:certificates.k8s.io:certificatesigningrequests:selfnodeclient
name: system:karmada:certificatesigningrequest:selfautoapprover
subjects:
- apiGroup: rbac.authorization.k8s.io
kind: Group
name: system:nodes
name: system:karmada:agents
---
# ClusterRole `system:karmada:agent-rbac-generator` is not used for the connection between the karmada-agent and the control plane,
# but is used by karmadactl register to generate the RBAC resources required by the karmada-agent.
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
labels:
karmada.io/bootstrapping: rbac-defaults
name: system:karmada:agent
name: system:karmada:agent-rbac-generator
rules:
- apiGroups:
- cluster.karmada.io
resources:
- clusters
verbs:
- create
- get
- list
- watch
- delete
- apiGroups:
- cluster.karmada.io
resources:
- clusters/status
verbs:
- update
- apiGroups:
- work.karmada.io
resources:
- works
verbs:
- create
- get
- list
- watch
- update
- delete
- apiGroups:
- work.karmada.io
resources:
- works/status
verbs:
- patch
- update
- apiGroups:
- config.karmada.io
resources:
- resourceinterpreterwebhookconfigurations
- resourceinterpretercustomizations
verbs:
- get
- list
- watch
- apiGroups:
- ""
resources:
- namespaces
verbs:
- get
- apiGroups:
- ""
resources:
- secrets
verbs:
- get
- create
- patch
- apiGroups:
- coordination.k8s.io
resources:
- leases
verbs:
- create
- get
- update
- apiGroups:
- certificates.k8s.io
resources:
- certificatesigningrequests
verbs:
- create
- get
- apiGroups:
- ""
resources:
- events
verbs:
- create
- patch
- update
- apiGroups: ['*']
resources: ['*']
verbs: ['*']
---
# User `system:karmada:agent:rbac-generator` is specifically used during the `karmadactl register` process to generate restricted RBAC resources for the `karmada-agent`.
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
labels:
karmada.io/bootstrapping: rbac-defaults
name: system:karmada:agent
name: system:karmada:agent-rbac-generator
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:karmada:agent
name: system:karmada:agent-rbac-generator
subjects:
- apiGroup: rbac.authorization.k8s.io
kind: Group
name: system:nodes
# To ensure the agent has the minimal RBAC permissions, the ideal approach is to
# use different RBAC configurations for different agents of member clusters with pull mode.
# Below is the minimal set of RBAC permissions required for a single pull mode member cluster.
# Here are the definitions of the variables used:
#
# - clustername: the name of the member cluster.
# - cluster_namespace: the namespace where the member cluster secrets are stored, default to karmada-cluster.
#
# ---
# apiVersion: rbac.authorization.k8s.io/v1
# kind: ClusterRole
# metadata:
# name: system:karmada:agent
# rules:
# - apiGroups:
# - cluster.karmada.io
# resources:
# - clusters
# resourceNames:
# - {{clustername}}
# verbs:
# - create
# - get
# - delete
# - apiGroups:
# - cluster.karmada.io
# resources:
# - clusters
# verbs:
# - list
# - watch
# - apiGroups:
# - cluster.karmada.io
# resources:
# - clusters/status
# resourceNames:
# - {{clustername}}
# verbs:
# - update
# - apiGroups:
# - config.karmada.io
# resources:
# - resourceinterpreterwebhookconfigurations
# - resourceinterpretercustomizations
# verbs:
# - get
# - list
# - watch
# - apiGroups:
# - ""
# resources:
# - namespaces
# verbs:
# - get
# - apiGroups:
# - coordination.k8s.io
# resources:
# - leases
# verbs:
# - create
# - get
# - update
# - apiGroups:
# - certificates.k8s.io
# resources:
# - certificatesigningrequests
# verbs:
# - create
# - get
# - apiGroups:
# - ""
# resources:
# - events
# verbs:
# - create
# - patch
# - update
#
# ---
# apiVersion: rbac.authorization.k8s.io/v1
# kind: ClusterRoleBinding
# metadata:
# name: system:karmada:agent
# roleRef:
# apiGroup: rbac.authorization.k8s.io
# kind: ClusterRole
# name: system:karmada:agent
# subjects:
# - apiGroup: rbac.authorization.k8s.io
# kind: Group
# name: system:nodes
#
# ---
# apiVersion: rbac.authorization.k8s.io/v1
# kind: Role
# metadata:
# name: system:karmada:agent-secret
# namespace: "{{cluster_namespace}}"
# rules:
# - apiGroups:
# - ""
# resources:
# - secrets
# resourceNames:
# - {{clustername}}-impersonator
# - {{clustername}}
# verbs:
# - get
# - create
# - patch
#
# ---
# apiVersion: rbac.authorization.k8s.io/v1
# kind: RoleBinding
# metadata:
# name: system:karmada:agent-secret
# namespace: "{{cluster_namespace}}"
# roleRef:
# apiGroup: rbac.authorization.k8s.io
# kind: Role
# name: system:karmada:agent-secret
# subjects:
# - apiGroup: rbac.authorization.k8s.io
# kind: Group
# name: system:nodes
#
# ---
# apiVersion: rbac.authorization.k8s.io/v1
# kind: Role
# metadata:
# name: system:karmada:agent-work
# namespace: "karmada-es-{{clustername}}"
# rules:
# - apiGroups:
# - work.karmada.io
# resources:
# - works
# verbs:
# - create
# - get
# - list
# - watch
# - update
# - delete
# - apiGroups:
# - work.karmada.io
# resources:
# - works/status
# verbs:
# - patch
# - update
#
# ---
# apiVersion: rbac.authorization.k8s.io/v1
# kind: RoleBinding
# metadata:
# name: system:karmada:agent-work
# namespace: "karmada-es-{{clustername}}"
# roleRef:
# apiGroup: rbac.authorization.k8s.io
# kind: Role
# name: system:karmada:agent-work
# subjects:
# - apiGroup: rbac.authorization.k8s.io
# kind: Group
# name: system:nodes
- apiGroup: rbac.authorization.k8s.io
kind: User
name: system:karmada:agent:rbac-generator

View File

@ -75,6 +75,22 @@ subjects:
name: system:bootstrappers:karmada:default-cluster-token
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: system:karmada:certificatesigningrequest:autoapprover
{{- if "karmada.commonLabels" }}
labels:
{{- include "karmada.commonLabels" . | nindent 4 }}
{{- end }}
rules:
- apiGroups:
- certificates.k8s.io
resources:
- certificatesigningrequests/clusteragent
verbs:
- create
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: system:karmada:agent-autoapprove-bootstrap
@ -85,13 +101,29 @@ metadata:
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:certificates.k8s.io:certificatesigningrequests:nodeclient
name: system:karmada:certificatesigningrequest:autoapprover
subjects:
- apiGroup: rbac.authorization.k8s.io
kind: Group
name: system:bootstrappers:karmada:default-cluster-token
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: system:karmada:certificatesigningrequest:selfautoapprover
{{- if "karmada.commonLabels" }}
labels:
{{- include "karmada.commonLabels" . | nindent 4 }}
{{- end }}
rules:
- apiGroups:
- certificates.k8s.io
resources:
- certificatesigningrequests/selfclusteragent
verbs:
- create
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: system:karmada:agent-autoapprove-certificate-rotation
@ -102,124 +134,32 @@ metadata:
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:certificates.k8s.io:certificatesigningrequests:selfnodeclient
name: system:karmada:certificatesigningrequest:selfautoapprover
subjects:
- apiGroup: rbac.authorization.k8s.io
kind: Group
name: system:nodes
name: system:karmada:agents
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: system:karmada:agent
name: system:karmada:agent-rbac-generator
{{- if "karmada.commonLabels" }}
labels:
{{- include "karmada.commonLabels" . | nindent 4 }}
{{- end }}
rules:
- apiGroups:
- authentication.k8s.io
- "*"
resources:
- tokenreviews
- "*"
verbs:
- create
- apiGroups:
- cluster.karmada.io
resources:
- clusters
verbs:
- create
- get
- list
- watch
- patch
- update
- delete
- apiGroups:
- cluster.karmada.io
resources:
- clusters/status
verbs:
- patch
- update
- apiGroups:
- work.karmada.io
resources:
- works
verbs:
- create
- get
- list
- watch
- update
- delete
- apiGroups:
- work.karmada.io
resources:
- works/status
verbs:
- patch
- update
- apiGroups:
- config.karmada.io
resources:
- resourceinterpreterwebhookconfigurations
- resourceinterpretercustomizations
verbs:
- get
- list
- watch
- apiGroups:
- ""
resources:
- namespaces
verbs:
- get
- list
- watch
- create
- apiGroups:
- ""
resources:
- secrets
verbs:
- get
- list
- watch
- create
- patch
- apiGroups:
- coordination.k8s.io
resources:
- leases
verbs:
- create
- delete
- get
- patch
- update
- apiGroups:
- certificates.k8s.io
resources:
- certificatesigningrequests
verbs:
- create
- get
- list
- watch
- apiGroups:
- ""
resources:
- events
verbs:
- create
- patch
- update
- "*"
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: system:karmada:agent
name: system:karmada:agent-rbac-generator
{{- if "karmada.commonLabels" }}
labels:
{{- include "karmada.commonLabels" . | nindent 4 }}
@ -227,9 +167,9 @@ metadata:
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:karmada:agent
name: system:karmada:agent-rbac-generator
subjects:
- apiGroup: rbac.authorization.k8s.io
kind: Group
name: system:nodes
kind: User
name: system:karmada:agent:rbac-generator
{{- end -}}

View File

@ -31,15 +31,15 @@ const (
// KarmadaAgentBootstrap defines the name of the ClusterRoleBinding that lets Karmada Agent post CSRs
KarmadaAgentBootstrap = "system:karmada:agent-bootstrap"
// KarmadaAgentGroup defines the group of Karmada Agent
KarmadaAgentGroup = "system:nodes"
KarmadaAgentGroup = "system:karmada:agents"
// KarmadaAgentAutoApproveBootstrapClusterRoleBinding defines the name of the ClusterRoleBinding that makes the csrapprover approve agent CSRs
KarmadaAgentAutoApproveBootstrapClusterRoleBinding = "system:karmada:agent-autoapprove-bootstrap"
// KarmadaAgentAutoApproveCertificateRotationClusterRoleBinding defines name of the ClusterRoleBinding that makes the csrapprover approve agent auto rotated CSRs
KarmadaAgentAutoApproveCertificateRotationClusterRoleBinding = "system:karmada:agent-autoapprove-certificate-rotation"
// CSRAutoApprovalClusterRoleName defines the name of the auto-bootstrapped ClusterRole for making the csrapprover controller auto-approve the CSR
CSRAutoApprovalClusterRoleName = "system:certificates.k8s.io:certificatesigningrequests:nodeclient"
CSRAutoApprovalClusterRoleName = "system:karmada:certificatesigningrequest:autoapprover"
// KarmadaAgentSelfCSRAutoApprovalClusterRoleName is a role for automatic CSR approvals for automatically rotated agent certificates
KarmadaAgentSelfCSRAutoApprovalClusterRoleName = "system:certificates.k8s.io:certificatesigningrequests:selfnodeclient"
KarmadaAgentSelfCSRAutoApprovalClusterRoleName = "system:karmada:certificatesigningrequest:selfautoapprover"
// KarmadaAgentBootstrapTokenAuthGroup specifies which group a Karmada Agent Bootstrap Token should be authenticated in
KarmadaAgentBootstrapTokenAuthGroup = "system:bootstrappers:karmada:default-cluster-token"
)
@ -60,7 +60,18 @@ func AllowBootstrapTokensToPostCSRs(clientSet kubernetes.Interface) error {
// AutoApproveKarmadaAgentBootstrapTokens creates RBAC rules in a way that makes Karmada Agent Bootstrap Tokens' CSR auto-approved by the csrapprover controller
func AutoApproveKarmadaAgentBootstrapTokens(clientSet kubernetes.Interface) error {
klog.Infoln("[bootstrap-token] configured RBAC rules to allow the csrapprover controller automatically approve CSRs from a Karmada Agent Bootstrap Token")
klog.Infoln("[bootstrap-token] configured RBAC rules to allow the agentcsrapproving controller automatically approve CSRs from a Karmada Agent Bootstrap Token")
csrAutoApprovalClusterRole := utils.ClusterRoleFromRules(CSRAutoApprovalClusterRoleName, []rbacv1.PolicyRule{
{
APIGroups: []string{"certificates.k8s.io"},
Resources: []string{"certificatesigningrequests/clusteragent"},
Verbs: []string{"create"},
},
}, nil, nil)
err := cmdutil.CreateOrUpdateClusterRole(clientSet, csrAutoApprovalClusterRole)
if err != nil {
return err
}
clusterRoleBinding := utils.ClusterRoleBindingFromSubjects(KarmadaAgentAutoApproveBootstrapClusterRoleBinding, CSRAutoApprovalClusterRoleName,
[]rbacv1.Subject{
@ -75,6 +86,17 @@ func AutoApproveKarmadaAgentBootstrapTokens(clientSet kubernetes.Interface) erro
// AutoApproveAgentCertificateRotation creates RBAC rules in a way that makes Agent certificate rotation CSR auto-approved by the csrapprover controller
func AutoApproveAgentCertificateRotation(clientSet kubernetes.Interface) error {
klog.Infoln("[bootstrap-token] configured RBAC rules to allow certificate rotation for all agent client certificates in the member cluster")
karmadaAgentSelfCSRAutoApprovalClusterRole := utils.ClusterRoleFromRules(KarmadaAgentSelfCSRAutoApprovalClusterRoleName, []rbacv1.PolicyRule{
{
APIGroups: []string{"certificates.k8s.io"},
Resources: []string{"certificatesigningrequests/selfclusteragent"},
Verbs: []string{"create"},
},
}, nil, nil)
err := cmdutil.CreateOrUpdateClusterRole(clientSet, karmadaAgentSelfCSRAutoApprovalClusterRole)
if err != nil {
return err
}
clusterRoleBinding := utils.ClusterRoleBindingFromSubjects(KarmadaAgentAutoApproveCertificateRotationClusterRoleBinding, KarmadaAgentSelfCSRAutoApprovalClusterRoleName,
[]rbacv1.Subject{

View File

@ -185,8 +185,8 @@ func createExtraResources(clientSet *kubernetes.Clientset, dir string) error {
return fmt.Errorf("error creating clusterinfo RBAC rules: %v", err)
}
// grant limited access permission to 'karmada-agent'
if err := grantAccessPermissionToAgent(clientSet); err != nil {
// grant access permission to 'karmada-agent-rbac-generator'
if err := grantAccessPermissionToAgentRBACGenerator(clientSet); err != nil {
return err
}

View File

@ -26,10 +26,11 @@ import (
)
const (
karmadaViewClusterRole = "karmada-view"
karmadaEditClusterRole = "karmada-edit"
karmadaAgentAccessClusterRole = "system:karmada:agent"
karmadaAgentGroup = "system:nodes"
karmadaViewClusterRole = "karmada-view"
karmadaEditClusterRole = "karmada-edit"
karmadaAgentRBACGeneratorClusterRole = "system:karmada:agent-rbac-generator"
karmadaAgentRBACGeneratorClusterRoleBinding = "system:karmada:agent-rbac-generator"
agentRBACGenerator = "system:karmada:agent:rbac-generator"
)
// grantProxyPermissionToAdmin grants the proxy permission to "system:admin"
@ -62,63 +63,13 @@ func grantProxyPermissionToAdmin(clientSet kubernetes.Interface) error {
return nil
}
// grantAccessPermissionToAgent grants the limited access permission to 'karmada-agent'
func grantAccessPermissionToAgent(clientSet kubernetes.Interface) error {
clusterRole := utils.ClusterRoleFromRules(karmadaAgentAccessClusterRole, []rbacv1.PolicyRule{
// grantAccessPermissionToAgentRBACGenerator grants the access permission to 'karmada-agent-rbac-generator'
func grantAccessPermissionToAgentRBACGenerator(clientSet kubernetes.Interface) error {
clusterRole := utils.ClusterRoleFromRules(karmadaAgentRBACGeneratorClusterRole, []rbacv1.PolicyRule{
{
APIGroups: []string{"authentication.k8s.io"},
Resources: []string{"tokenreviews"},
Verbs: []string{"create"},
},
{
APIGroups: []string{"cluster.karmada.io"},
Resources: []string{"clusters"},
Verbs: []string{"create", "get", "list", "watch", "patch", "update", "delete"},
},
{
APIGroups: []string{"cluster.karmada.io"},
Resources: []string{"clusters/status"},
Verbs: []string{"patch", "update"},
},
{
APIGroups: []string{"work.karmada.io"},
Resources: []string{"works"},
Verbs: []string{"create", "get", "list", "watch", "update", "delete"},
},
{
APIGroups: []string{"work.karmada.io"},
Resources: []string{"works/status"},
Verbs: []string{"patch", "update"},
},
{
APIGroups: []string{"config.karmada.io"},
Resources: []string{"resourceinterpreterwebhookconfigurations", "resourceinterpretercustomizations"},
Verbs: []string{"get", "list", "watch"},
},
{
APIGroups: []string{""},
Resources: []string{"namespaces"},
Verbs: []string{"get", "list", "watch", "create"},
},
{
APIGroups: []string{""},
Resources: []string{"secrets"},
Verbs: []string{"get", "list", "watch", "create", "patch"},
},
{
APIGroups: []string{"coordination.k8s.io"},
Resources: []string{"leases"},
Verbs: []string{"create", "delete", "get", "patch", "update"},
},
{
APIGroups: []string{"certificates.k8s.io"},
Resources: []string{"certificatesigningrequests"},
Verbs: []string{"create", "get", "list", "watch"},
},
{
APIGroups: []string{""},
Resources: []string{"events"},
Verbs: []string{"create", "patch", "update"},
APIGroups: []string{"*"},
Resources: []string{"*"},
Verbs: []string{"*"},
},
}, nil, nil)
err := cmdutil.CreateOrUpdateClusterRole(clientSet, clusterRole)
@ -126,14 +77,14 @@ func grantAccessPermissionToAgent(clientSet kubernetes.Interface) error {
return err
}
clusterRoleBinding := utils.ClusterRoleBindingFromSubjects(karmadaAgentAccessClusterRole, karmadaAgentAccessClusterRole,
clusterRoleBinding := utils.ClusterRoleBindingFromSubjects(karmadaAgentRBACGeneratorClusterRoleBinding, karmadaAgentRBACGeneratorClusterRole,
[]rbacv1.Subject{
{
Kind: rbacv1.GroupKind,
Name: karmadaAgentGroup,
Kind: rbacv1.UserKind,
Name: agentRBACGenerator,
}}, nil)
klog.V(1).Info("Grant the limited access permission to 'karmada-agent'")
klog.V(1).Info("Grant the access permission to 'karmada-agent-rbac-generator'")
err = cmdutil.CreateOrUpdateClusterRoleBinding(clientSet, clusterRoleBinding)
if err != nil {
return err

View File

@ -31,8 +31,8 @@ func Test_grantProxyPermissionToAdmin(t *testing.T) {
func Test_grantAccessPermissionToAgent(t *testing.T) {
client := fake.NewSimpleClientset()
if err := grantAccessPermissionToAgent(client); err != nil {
t.Errorf("grantAccessPermissionToAgent() expected no error, but got err: %v", err)
if err := grantAccessPermissionToAgentRBACGenerator(client); err != nil {
t.Errorf("grantAccessPermissionToAgentRBACGenerator() expected no error, but got err: %v", err)
}
}

View File

@ -51,7 +51,6 @@ import (
"github.com/karmada-io/karmada/pkg/apis/cluster/validation"
karmadaclientset "github.com/karmada-io/karmada/pkg/generated/clientset/versioned"
addonutils "github.com/karmada-io/karmada/pkg/karmadactl/addons/utils"
"github.com/karmada-io/karmada/pkg/karmadactl/options"
cmdutil "github.com/karmada-io/karmada/pkg/karmadactl/util"
"github.com/karmada-io/karmada/pkg/karmadactl/util/apiclient"
@ -82,9 +81,11 @@ const (
// CACertPath defines default location of CA certificate on Linux
CACertPath = "/etc/karmada/pki/ca.crt"
// ClusterPermissionPrefix defines the common name of karmada agent certificate
ClusterPermissionPrefix = "system:node:"
ClusterPermissionPrefix = "system:karmada:agent:"
// ClusterPermissionGroups defines the organization of karmada agent certificate
ClusterPermissionGroups = "system:nodes"
ClusterPermissionGroups = "system:karmada:agents"
// AgentRBACGenerator defines the common name of karmada agent rbac generator certificate
AgentRBACGenerator = "system:karmada:agent:rbac-generator"
// KarmadaAgentBootstrapKubeConfigFileName defines the file name for the kubeconfig that the karmada-agent will use to do
// the TLS bootstrap to get itself an unique credential
KarmadaAgentBootstrapKubeConfigFileName = "bootstrap-karmada-agent.conf"
@ -97,8 +98,8 @@ const (
KarmadaAgentName = "karmada-agent"
// KarmadaAgentServiceAccountName is the name of karmada-agent serviceaccount
KarmadaAgentServiceAccountName = "karmada-agent-sa"
// SignerName defines the signer name for csr, 'kubernetes.io/kube-apiserver-client-kubelet' can sign the csr automatically
SignerName = "kubernetes.io/kube-apiserver-client-kubelet"
// SignerName defines the signer name for csr, 'kubernetes.io/kube-apiserver-client' can sign the csr with `O=system:agents,CN=system:agent:` automatically if agentcsrapproving controller if enabled.
SignerName = "kubernetes.io/kube-apiserver-client"
// BootstrapUserName defines bootstrap user name
BootstrapUserName = "token-bootstrap-client"
// DefaultClusterName defines the default cluster name
@ -260,6 +261,9 @@ type CommandRegisterOption struct {
memberClusterEndpoint string
memberClusterClient *kubeclient.Clientset
// rbacResources contains RBAC resources that grant the necessary permissions for pull mode cluster to access to Karmada control plane.
rbacResources *RBACResources
}
// Complete ensures that options are valid and marshals them if necessary.
@ -288,6 +292,8 @@ func (o *CommandRegisterOption) Complete(args []string) error {
o.ClusterName = config.Contexts[config.CurrentContext].Cluster
}
o.rbacResources = GenerateRBACResources(o.ClusterName, o.ClusterNamespace)
o.memberClusterEndpoint = restConfig.Host
o.memberClusterClient, err = apiclient.NewClientSet(restConfig)
@ -358,6 +364,74 @@ func (o *CommandRegisterOption) Run(parentCommand string) error {
return err
}
var rbacClient *kubeclient.Clientset
defer func() {
if err != nil && rbacClient != nil {
fmt.Println("karmadactl register failed and started deleting the created resources")
err = o.rbacResources.Delete(rbacClient)
if err != nil {
klog.Warningf("Failed to delete rbac resources: %v", err)
}
}
}()
rbacClient, err = o.EnsureNecessaryResourcesExistInControlPlane(bootstrapClient, karmadaClusterInfo)
if err != nil {
return err
}
err = o.EnsureNecessaryResourcesExistInMemberCluster(bootstrapClient, karmadaClusterInfo)
if err != nil {
return err
}
fmt.Printf("\ncluster(%s) is joined successfully\n", o.ClusterName)
return nil
}
// EnsureNecessaryResourcesExistInControlPlane ensures that all necessary resources are exist in Karmada control plane.
func (o *CommandRegisterOption) EnsureNecessaryResourcesExistInControlPlane(bootstrapClient *kubeclient.Clientset, karmadaClusterInfo *clientcmdapi.Cluster) (*kubeclient.Clientset, error) {
csrName := "agent-rbac-generator-" + o.ClusterName + k8srand.String(5)
rbacCfg, err := o.constructAgentRBACGeneratorConfig(bootstrapClient, karmadaClusterInfo, csrName)
if err != nil {
return nil, err
}
kubelient, err := ToClientSet(rbacCfg)
if err != nil {
return nil, err
}
defer func() {
err = kubelient.CertificatesV1().CertificateSigningRequests().Delete(context.Background(), csrName, metav1.DeleteOptions{})
if err != nil {
klog.Warningf("Failed to delete CertificateSigningRequests %s: %v", csrName, err)
}
}()
fmt.Println("[karmada-agent-start] Waiting to check cluster exists")
karmadaClient, err := ToKarmadaClient(rbacCfg)
if err != nil {
return kubelient, err
}
_, exist, err := karmadautil.GetClusterWithKarmadaClient(karmadaClient, o.ClusterName)
if err != nil {
return kubelient, err
} else if exist {
return kubelient, fmt.Errorf("failed to register as cluster with name %s already exists", o.ClusterName)
}
fmt.Println("[karmada-agent-start] Assign the necessary RBAC permissions to the agent")
err = o.ensureAgentRBACResourcesExistInControlPlane(kubelient)
if err != nil {
return kubelient, err
}
return kubelient, nil
}
// EnsureNecessaryResourcesExistInMemberCluster ensures that all necessary resources are exist in the registering cluster.
func (o *CommandRegisterOption) EnsureNecessaryResourcesExistInMemberCluster(bootstrapClient *kubeclient.Clientset, karmadaClusterInfo *clientcmdapi.Cluster) error {
// construct the final kubeconfig file used by karmada agent to connect to karmada apiserver
fmt.Println("[karmada-agent-start] Waiting to construct karmada-agent kubeconfig")
karmadaAgentCfg, err := o.constructKarmadaAgentConfig(bootstrapClient, karmadaClusterInfo)
@ -365,46 +439,32 @@ func (o *CommandRegisterOption) Run(parentCommand string) error {
return err
}
fmt.Println("[karmada-agent-start] Waiting to check cluster exists")
karmadaClient, err := ToKarmadaClient(karmadaAgentCfg)
if err != nil {
return err
}
_, exist, err := karmadautil.GetClusterWithKarmadaClient(karmadaClient, o.ClusterName)
if err != nil {
return err
} else if exist {
return fmt.Errorf("failed to register as cluster with name %s already exists", o.ClusterName)
}
// It's necessary to set the label of namespace to make sure that the namespace is created by Karmada.
labels := map[string]string{
karmadautil.KarmadaSystemLabel: karmadautil.KarmadaSystemLabelValue,
}
// ensure namespace where the karmada-agent resources be deployed exists in the member cluster
if _, err := karmadautil.EnsureNamespaceExistWithLabels(o.memberClusterClient, o.Namespace, o.DryRun, labels); err != nil {
if _, err = karmadautil.EnsureNamespaceExistWithLabels(o.memberClusterClient, o.Namespace, o.DryRun, labels); err != nil {
return err
}
// create the necessary secret and RBAC in the member cluster
fmt.Println("[karmada-agent-start] Waiting the necessary secret and RBAC")
if err := o.createSecretAndRBACInMemberCluster(karmadaAgentCfg); err != nil {
if err = o.createSecretAndRBACInMemberCluster(karmadaAgentCfg); err != nil {
return err
}
// create karmada-agent Deployment in the member cluster
fmt.Println("[karmada-agent-start] Waiting karmada-agent Deployment")
KarmadaAgentDeployment := o.makeKarmadaAgentDeployment()
if _, err := o.memberClusterClient.AppsV1().Deployments(o.Namespace).Create(context.TODO(), KarmadaAgentDeployment, metav1.CreateOptions{}); err != nil {
if _, err = o.memberClusterClient.AppsV1().Deployments(o.Namespace).Create(context.TODO(), KarmadaAgentDeployment, metav1.CreateOptions{}); err != nil {
return err
}
if err := addonutils.WaitForDeploymentRollout(o.memberClusterClient, KarmadaAgentDeployment, int(o.Timeout)); err != nil {
if err = cmdutil.WaitForDeploymentRollout(o.memberClusterClient, KarmadaAgentDeployment, int(o.Timeout)); err != nil {
return err
}
fmt.Printf("\ncluster(%s) is joined successfully\n", o.ClusterName)
return nil
}
@ -505,11 +565,316 @@ func (o *CommandRegisterOption) discoveryBootstrapConfigAndClusterInfo(bootstrap
return bootstrapClient, clusterinfo, nil
}
// constructKarmadaAgentConfig construct the final kubeconfig used by karmada-agent
func (o *CommandRegisterOption) constructKarmadaAgentConfig(bootstrapClient *kubeclient.Clientset, karmadaClusterInfo *clientcmdapi.Cluster) (*clientcmdapi.Config, error) {
// ensureAgentRBACResourcesExistInControlPlane ensures that necessary RBAC resources for karmada-agent are exist in control plane.
func (o *CommandRegisterOption) ensureAgentRBACResourcesExistInControlPlane(client kubeclient.Interface) error {
for i := range o.rbacResources.ClusterRoles {
_, err := karmadautil.CreateClusterRole(client, o.rbacResources.ClusterRoles[i])
if err != nil {
return err
}
}
for i := range o.rbacResources.ClusterRoleBindings {
_, err := karmadautil.CreateClusterRoleBinding(client, o.rbacResources.ClusterRoleBindings[i])
if err != nil {
return err
}
}
for i := range o.rbacResources.Roles {
roleNamespace := &corev1.Namespace{
ObjectMeta: metav1.ObjectMeta{
Name: o.rbacResources.Roles[i].GetNamespace(),
Labels: map[string]string{
karmadautil.KarmadaSystemLabel: karmadautil.KarmadaSystemLabelValue,
},
},
}
_, err := karmadautil.CreateNamespace(client, roleNamespace)
if err != nil {
return err
}
_, err = karmadautil.CreateRole(client, o.rbacResources.Roles[i])
if err != nil {
return err
}
}
for i := range o.rbacResources.RoleBindings {
_, err := karmadautil.CreateRoleBinding(client, o.rbacResources.RoleBindings[i])
if err != nil {
return err
}
}
return nil
}
// RBACResources defines the list of rbac resources.
type RBACResources struct {
ClusterRoles []*rbacv1.ClusterRole
ClusterRoleBindings []*rbacv1.ClusterRoleBinding
Roles []*rbacv1.Role
RoleBindings []*rbacv1.RoleBinding
}
// GenerateRBACResources generates rbac resources.
func GenerateRBACResources(clusterName, clusterNamespace string) *RBACResources {
return &RBACResources{
ClusterRoles: []*rbacv1.ClusterRole{GenerateClusterRole(clusterName)},
ClusterRoleBindings: []*rbacv1.ClusterRoleBinding{GenerateClusterRoleBinding(clusterName)},
Roles: []*rbacv1.Role{GenerateSecretAccessRole(clusterName, clusterNamespace), GenerateWorkAccessRole(clusterName)},
RoleBindings: []*rbacv1.RoleBinding{GenerateSecretAccessRoleBinding(clusterName, clusterNamespace), GenerateWorkAccessRoleBinding(clusterName)},
}
}
// List return the list of rbac resources.
func (r *RBACResources) List() []Obj {
var obj []Obj
for i := range r.ClusterRoles {
obj = append(obj, Obj{Kind: "ClusterRole", Name: r.ClusterRoles[i].GetName()})
}
for i := range r.ClusterRoleBindings {
obj = append(obj, Obj{Kind: "ClusterRoleBinding", Name: r.ClusterRoleBindings[i].GetName()})
}
for i := range r.Roles {
obj = append(obj, Obj{Kind: "Role", Name: r.Roles[i].GetName(), Namespace: r.Roles[i].GetNamespace()})
}
for i := range r.RoleBindings {
obj = append(obj, Obj{Kind: "RoleBinding", Name: r.RoleBindings[i].GetName(), Namespace: r.RoleBindings[i].GetNamespace()})
}
return obj
}
// ToString returns a list of RBAC resources in string format.
func (r *RBACResources) ToString() string {
var resources []string
for i := range r.List() {
resources = append(resources, r.List()[i].ToString())
}
return strings.Join(resources, "\n")
}
// Delete deletes RBAC resources.
func (r *RBACResources) Delete(client kubeclient.Interface) error {
var err error
for _, resource := range r.List() {
switch resource.Kind {
case "ClusterRole":
err = karmadautil.DeleteClusterRole(client, resource.Name)
case "ClusterRoleBinding":
err = karmadautil.DeleteClusterRoleBinding(client, resource.Name)
case "Role":
err = karmadautil.DeleteRole(client, resource.Namespace, resource.Name)
case "RoleBinding":
err = karmadautil.DeleteRoleBinding(client, resource.Namespace, resource.Name)
}
if err != nil {
return err
}
}
return nil
}
// Obj defines the struct which contains the information of kind, name and namespace.
type Obj struct{ Kind, Name, Namespace string }
// ToString returns a string that concatenates kind, name, and namespace using "/".
func (o *Obj) ToString() string {
if o.Namespace == "" {
return fmt.Sprintf("%s/%s", o.Kind, o.Name)
}
return fmt.Sprintf("%s/%s/%s", o.Kind, o.Namespace, o.Name)
}
// GenerateClusterRole generates the clusterRole that karmada-agent needed.
func GenerateClusterRole(clusterName string) *rbacv1.ClusterRole {
clusterRoleName := fmt.Sprintf("system:karmada:%s:agent", clusterName)
return &rbacv1.ClusterRole{
TypeMeta: metav1.TypeMeta{
APIVersion: rbacv1.SchemeGroupVersion.String(),
Kind: "ClusterRole",
},
ObjectMeta: metav1.ObjectMeta{
Name: clusterRoleName,
},
Rules: []rbacv1.PolicyRule{
{
APIGroups: []string{"cluster.karmada.io"},
Resources: []string{"clusters"},
ResourceNames: []string{clusterName},
Verbs: []string{"get", "delete"},
},
{
APIGroups: []string{"cluster.karmada.io"},
Resources: []string{"clusters"},
Verbs: []string{"create", "list", "watch"},
},
{
APIGroups: []string{"cluster.karmada.io"},
Resources: []string{"clusters/status"},
ResourceNames: []string{clusterName},
Verbs: []string{"update"},
},
{
APIGroups: []string{"config.karmada.io"},
Resources: []string{"resourceinterpreterwebhookconfigurations", "resourceinterpretercustomizations"},
Verbs: []string{"get", "list", "watch"},
},
{
APIGroups: []string{""},
Resources: []string{"namespaces"},
Verbs: []string{"get"},
},
{
APIGroups: []string{"coordination.k8s.io"},
Resources: []string{"leases"},
Verbs: []string{"get", "create", "update"},
},
{
APIGroups: []string{"certificates.k8s.io"},
Resources: []string{"certificatesigningrequests"},
Verbs: []string{"get", "create"},
},
{
APIGroups: []string{""},
Resources: []string{"services"},
Verbs: []string{"list", "watch"},
},
{
APIGroups: []string{""},
Resources: []string{"events"},
Verbs: []string{"patch", "create", "update"},
},
},
}
}
// GenerateClusterRoleBinding generates the clusterRoleBinding that karmada-agent needed.
func GenerateClusterRoleBinding(clusterName string) *rbacv1.ClusterRoleBinding {
return &rbacv1.ClusterRoleBinding{
TypeMeta: metav1.TypeMeta{
APIVersion: "rbac.authorization.k8s.io/v1",
Kind: "ClusterRoleBinding",
},
ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("system:karmada:%s:agent", clusterName)},
Subjects: []rbacv1.Subject{
{
APIGroup: "rbac.authorization.k8s.io",
Kind: "User",
Name: generateAgentUserName(clusterName),
},
},
RoleRef: rbacv1.RoleRef{
APIGroup: "rbac.authorization.k8s.io",
Kind: "ClusterRole",
Name: fmt.Sprintf("system:karmada:%s:agent", clusterName),
},
}
}
// GenerateSecretAccessRole generates the secret-related Role that karmada-agent needed.
func GenerateSecretAccessRole(clusterName, clusterNamespace string) *rbacv1.Role {
secretAccessRoleName := fmt.Sprintf("system:karmada:%s:agent-secret", clusterName)
return &rbacv1.Role{
ObjectMeta: metav1.ObjectMeta{
Name: secretAccessRoleName,
Namespace: clusterNamespace,
},
Rules: []rbacv1.PolicyRule{
{
Verbs: []string{"get", "patch"},
APIGroups: []string{""},
Resources: []string{"secrets"},
ResourceNames: []string{clusterName, clusterName + "-impersonator"},
},
{
Verbs: []string{"create"},
APIGroups: []string{""},
Resources: []string{"secrets"},
},
},
}
}
// GenerateSecretAccessRoleBinding generates the secret-related RoleBinding that karmada-agent needed.
func GenerateSecretAccessRoleBinding(clusterName, clusterNamespace string) *rbacv1.RoleBinding {
return &rbacv1.RoleBinding{
TypeMeta: metav1.TypeMeta{
APIVersion: "rbac.authorization.k8s.io/v1",
Kind: "RoleBinding",
},
ObjectMeta: metav1.ObjectMeta{
Name: fmt.Sprintf("system:karmada:%s:agent-secret", clusterName),
Namespace: clusterNamespace,
},
Subjects: []rbacv1.Subject{
{
APIGroup: "rbac.authorization.k8s.io",
Kind: "User",
Name: generateAgentUserName(clusterName),
},
},
RoleRef: rbacv1.RoleRef{
APIGroup: "rbac.authorization.k8s.io",
Kind: "Role",
Name: fmt.Sprintf("system:karmada:%s:agent-secret", clusterName),
},
}
}
// GenerateWorkAccessRole generates the work-related Role that karmada-agent needed.
func GenerateWorkAccessRole(clusterName string) *rbacv1.Role {
workAccessRoleName := fmt.Sprintf("system:karmada:%s:agent-work", clusterName)
return &rbacv1.Role{
ObjectMeta: metav1.ObjectMeta{
Name: workAccessRoleName,
Namespace: "karmada-es-" + clusterName,
},
Rules: []rbacv1.PolicyRule{
{
Verbs: []string{"get", "create", "list", "watch", "update", "delete"},
APIGroups: []string{"work.karmada.io"},
Resources: []string{"works"},
},
{
Verbs: []string{"patch", "update"},
APIGroups: []string{"work.karmada.io"},
Resources: []string{"works/status"},
},
},
}
}
// GenerateWorkAccessRoleBinding generates the work-related RoleBinding that karmada-agent needed.
func GenerateWorkAccessRoleBinding(clusterName string) *rbacv1.RoleBinding {
return &rbacv1.RoleBinding{
TypeMeta: metav1.TypeMeta{
APIVersion: "rbac.authorization.k8s.io/v1",
Kind: "RoleBinding",
},
ObjectMeta: metav1.ObjectMeta{
Name: fmt.Sprintf("system:karmada:%s:agent-work", clusterName),
Namespace: "karmada-es-" + clusterName,
},
Subjects: []rbacv1.Subject{
{
APIGroup: "rbac.authorization.k8s.io",
Kind: "User",
Name: generateAgentUserName(clusterName),
},
},
RoleRef: rbacv1.RoleRef{
APIGroup: "rbac.authorization.k8s.io",
Kind: "Role",
Name: fmt.Sprintf("system:karmada:%s:agent-work", clusterName),
},
}
}
func (o *CommandRegisterOption) constructKubeConfig(bootstrapClient *kubeclient.Clientset, karmadaClusterInfo *clientcmdapi.Cluster, csrName, commonName string, organization []string) (*clientcmdapi.Config, error) {
var cert []byte
pk, csr, err := generateKeyAndCSR(o.ClusterName)
pk, csr, err := generateKeyAndCSR(commonName, organization)
if err != nil {
return nil, err
}
@ -519,8 +884,6 @@ func (o *CommandRegisterOption) constructKarmadaAgentConfig(bootstrapClient *kub
return nil, err
}
csrName := o.ClusterName + "-" + k8srand.String(5)
certificateSigningRequest := &certificatesv1.CertificateSigningRequest{
ObjectMeta: metav1.ObjectMeta{
Name: csrName,
@ -547,35 +910,45 @@ func (o *CommandRegisterOption) constructKarmadaAgentConfig(bootstrapClient *kub
if err != nil {
return nil, err
}
klog.V(1).Infof("Waiting for the client certificate to be issued")
klog.V(1).Infof(fmt.Sprintf("Waiting for the client certificate %s to be issued", csrName))
err = wait.PollUntilContextTimeout(context.TODO(), 1*time.Second, o.Timeout, false, func(context.Context) (done bool, err error) {
csrOK, err := bootstrapClient.CertificatesV1().CertificateSigningRequests().Get(context.TODO(), csrName, metav1.GetOptions{})
if err != nil {
return false, fmt.Errorf("failed to get the cluster csr %s. err: %v", o.ClusterName, err)
return false, fmt.Errorf("failed to get the cluster csr %s. err: %v", csrName, err)
}
if csrOK.Status.Certificate != nil {
klog.V(1).Infof("Signing certificate successfully")
klog.V(1).Infof(fmt.Sprintf("Signing certificate of csr %s successfully", csrName))
cert = csrOK.Status.Certificate
return true, nil
}
klog.V(1).Infof("Waiting for the client certificate to be issued")
klog.V(1).Infof(fmt.Sprintf("Waiting for the client certificate of csr %s to be issued", csrName))
klog.V(1).Infof("Approve the CSR %s manually by executing `kubectl certificate approve %s` on the control plane", csrName, csrName)
return false, nil
})
if err != nil {
return nil, err
}
karmadaAgentCfg := CreateWithCert(
return CreateWithCert(
karmadaClusterInfo.Server,
DefaultClusterName,
o.ClusterName,
karmadaClusterInfo.CertificateAuthorityData,
cert,
pkData,
)
), nil
}
// constructKarmadaAgentConfig constructs the final kubeconfig used by karmada-agent
func (o *CommandRegisterOption) constructKarmadaAgentConfig(bootstrapClient *kubeclient.Clientset, karmadaClusterInfo *clientcmdapi.Cluster) (*clientcmdapi.Config, error) {
csrName := o.ClusterName + "-" + k8srand.String(5)
karmadaAgentCfg, err := o.constructKubeConfig(bootstrapClient, karmadaClusterInfo, csrName, generateAgentUserName(o.ClusterName), []string{ClusterPermissionGroups})
if err != nil {
return nil, err
}
kubeConfigFile := filepath.Join(KarmadaDir, KarmadaAgentKubeConfigFileName)
@ -588,6 +961,11 @@ func (o *CommandRegisterOption) constructKarmadaAgentConfig(bootstrapClient *kub
return karmadaAgentCfg, nil
}
// constructKarmadaAgentConfig constructs the kubeconfig to generate rbac config for karmada-agent.
func (o *CommandRegisterOption) constructAgentRBACGeneratorConfig(bootstrapClient *kubeclient.Clientset, karmadaClusterInfo *clientcmdapi.Cluster, csrName string) (*clientcmdapi.Config, error) {
return o.constructKubeConfig(bootstrapClient, karmadaClusterInfo, csrName, AgentRBACGenerator, []string{ClusterPermissionGroups})
}
// createSecretAndRBACInMemberCluster create required secrets and rbac in member cluster
func (o *CommandRegisterOption) createSecretAndRBACInMemberCluster(karmadaAgentCfg *clientcmdapi.Config) error {
configBytes, err := clientcmd.Write(*karmadaAgentCfg)
@ -781,7 +1159,7 @@ func (o *CommandRegisterOption) makeKarmadaAgentDeployment() *appsv1.Deployment
}
// generateKeyAndCSR generate private key and csr
func generateKeyAndCSR(clusterName string) (*rsa.PrivateKey, []byte, error) {
func generateKeyAndCSR(commonName string, organization []string) (*rsa.PrivateKey, []byte, error) {
pk, err := rsa.GenerateKey(rand.Reader, 3072)
if err != nil {
return nil, nil, err
@ -789,8 +1167,8 @@ func generateKeyAndCSR(clusterName string) (*rsa.PrivateKey, []byte, error) {
csr, err := x509.CreateCertificateRequest(rand.Reader, &x509.CertificateRequest{
Subject: pkix.Name{
CommonName: ClusterPermissionPrefix + clusterName,
Organization: []string{ClusterPermissionGroups},
CommonName: commonName,
Organization: organization,
},
}, pk)
if err != nil {
@ -1070,3 +1448,7 @@ func ToKarmadaClient(config *clientcmdapi.Config) (*karmadaclientset.Clientset,
return karmadaClient, nil
}
func generateAgentUserName(clusterName string) string {
return ClusterPermissionPrefix + clusterName
}

View File

@ -135,6 +135,9 @@ type CommandUnregisterOption struct {
// MemberClusterClient member cluster client set
MemberClusterClient kubeclient.Interface
// rbacResources contains RBAC resources that grant the necessary permissions for the unregistering cluster to access to Karmada control plane.
rbacResources *register.RBACResources
}
// AddFlags adds flags to the specified FlagSet.
@ -157,6 +160,8 @@ func (j *CommandUnregisterOption) Complete(args []string) error {
if len(args) > 0 {
j.ClusterName = args[0]
}
j.rbacResources = register.GenerateRBACResources(j.ClusterName, j.ClusterNamespace)
return nil
}
@ -303,30 +308,67 @@ func (j *CommandUnregisterOption) getKarmadaAgentConfig(agent *appsv1.Deployment
return clientcmd.Load(agentConfigSecret.Data[fileName])
}
type obj struct{ Kind, Name, Namespace string }
func (o *obj) ToString() string {
if o.Namespace == "" {
return fmt.Sprintf("%s/%s", o.Kind, o.Name)
}
return fmt.Sprintf("%s/%s/%s", o.Kind, o.Namespace, o.Name)
}
// RunUnregisterCluster unregister the pull mode cluster from karmada.
func (j *CommandUnregisterOption) RunUnregisterCluster() error {
if j.DryRun {
return nil
}
// 1. delete the cluster object from the Karmada control plane
start := time.Now()
// 1. delete the work object from the Karmada control plane
// When deleting a cluster, the deletion triggers the removal of executionSpace, which can lead to the deletion of RBAC roles related to work.
// Therefore, the deletion of work should be performed before deleting the cluster.
err := cmdutil.EnsureWorksDeleted(j.ControlPlaneClient, names.GenerateExecutionSpaceName(j.ClusterName), j.Wait)
if err != nil {
klog.Errorf("Failed to delete works object. cluster name: %s, error: %v", j.ClusterName, err)
return err
}
j.Wait = j.Wait - time.Since(start)
// 2. delete the cluster object from the Karmada control plane
//TODO: add flag --force to implement force deletion.
if err := cmdutil.DeleteClusterObject(j.ControlPlaneKubeClient, j.ControlPlaneClient, j.ClusterName, j.Wait, j.DryRun, false); err != nil {
if err = cmdutil.DeleteClusterObject(j.ControlPlaneKubeClient, j.ControlPlaneClient, j.ClusterName, j.Wait, j.DryRun, false); err != nil {
klog.Errorf("Failed to delete cluster object. cluster name: %s, error: %v", j.ClusterName, err)
return err
}
klog.Infof("Successfully delete cluster object (%s) from control plane.", j.ClusterName)
// 2. delete resource created by karmada in member cluster
if j.KarmadaConfig != "" {
if err = j.rbacResources.Delete(j.ControlPlaneKubeClient); err != nil {
klog.Errorf("Failed to delete karmada-agent RBAC resources from control plane. cluster name: %s, error: %v", j.ClusterName, err)
return err
}
klog.Infof("Successfully delete karmada-agent RBAC resources from control plane. cluster name: %s", j.ClusterName)
} else {
klog.Warningf("The RBAC resources on the control plane need to be manually cleaned up, including the following resources:\n%s", j.rbacResources.ToString())
}
// 3. delete resource created by karmada in member cluster
if err = j.cleanupMemberClusterResources(); err != nil {
return err
}
// 4. delete local obsolete files generated by karmadactl
localObsoleteFiles := []register.Obj{
{Kind: "File", Name: filepath.Join(register.KarmadaDir, register.KarmadaAgentKubeConfigFileName)},
{Kind: "File", Name: register.CACertPath},
}
for _, obj := range localObsoleteFiles {
if err = os.Remove(obj.Name); err != nil {
if os.IsNotExist(err) {
continue
}
klog.Errorf("Failed to delete local file (%v) in current node: %+v.", obj.Name, err)
return err
}
klog.Infof("Successfully delete local file (%v) in current node.", obj.Name)
}
return nil
}
// cleanupMemberClusterResources clean up the resources which created by karmada in member cluster
func (j *CommandUnregisterOption) cleanupMemberClusterResources() error {
var err error
for _, resource := range j.listMemberClusterResources() {
switch resource.Kind {
@ -350,29 +392,12 @@ func (j *CommandUnregisterOption) RunUnregisterCluster() error {
}
klog.Infof("Successfully delete resource (%v) from member cluster (%s).", resource, j.ClusterName)
}
// 3. delete local obsolete files generated by karmadactl
localObsoleteFiles := []obj{
{Kind: "File", Name: filepath.Join(register.KarmadaDir, register.KarmadaAgentKubeConfigFileName)},
{Kind: "File", Name: register.CACertPath},
}
for _, obj := range localObsoleteFiles {
if err = os.Remove(obj.Name); err != nil {
if os.IsNotExist(err) {
continue
}
klog.Errorf("Failed to delete local file (%v) in current node: %+v.", obj.Name, err)
return err
}
klog.Infof("Successfully delete local file (%v) in current node.", obj.Name)
}
return nil
return err
}
// listMemberClusterResources lists resources to be deleted which created by karmada in member cluster
func (j *CommandUnregisterOption) listMemberClusterResources() []obj {
return []obj{
func (j *CommandUnregisterOption) listMemberClusterResources() []register.Obj {
return []register.Obj{
// the rbac resource prepared for karmada-controller-manager to access member cluster's kube-apiserver
{Kind: "ServiceAccount", Namespace: j.ClusterNamespace, Name: names.GenerateServiceAccountName(j.ClusterName)},
{Kind: "ClusterRole", Name: names.GenerateRoleName(names.GenerateServiceAccountName(j.ClusterName))},

View File

@ -216,6 +216,7 @@ func TestCommandUnregisterOption_RunUnregisterCluster(t *testing.T) {
}
j.ControlPlaneClient = fakekarmadaclient.NewSimpleClientset(tt.clusterObject...)
j.MemberClusterClient = fake.NewSimpleClientset(tt.clusterResources...)
j.rbacResources = register.GenerateRBACResources(j.ClusterName, j.ClusterNamespace)
err := j.RunUnregisterCluster()
if (err == nil && tt.wantErr) || (err != nil && !tt.wantErr) {
t.Errorf("RunUnregisterCluster() error = %v, wantErr %v", err, tt.wantErr)

View File

@ -0,0 +1,55 @@
/*
Copyright 2024 The Karmada Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package util
import (
"context"
"fmt"
"time"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/wait"
karmadaclientset "github.com/karmada-io/karmada/pkg/generated/clientset/versioned"
)
// EnsureWorksDeleted ensures that all Work resources in the specified namespace are deleted.
func EnsureWorksDeleted(controlPlaneKarmadaClient karmadaclientset.Interface, namespace string,
timeout time.Duration) error {
// make sure the works object under the given namespace has been deleted.
err := wait.PollUntilContextTimeout(context.TODO(), 1*time.Second, timeout, false, func(context.Context) (done bool, err error) {
list, err := controlPlaneKarmadaClient.WorkV1alpha1().Works(namespace).List(context.TODO(), metav1.ListOptions{})
if err != nil {
return false, fmt.Errorf("failed to list work in namespace %s", namespace)
}
if len(list.Items) == 0 {
return true, nil
}
for i := range list.Items {
work := &list.Items[i]
err = controlPlaneKarmadaClient.WorkV1alpha1().Works(namespace).Delete(context.TODO(), work.GetName(), metav1.DeleteOptions{})
if err != nil && !apierrors.IsNotFound(err) {
return false, fmt.Errorf("failed to delete the work(%s/%s)", namespace, work.GetName())
}
}
return false, nil
})
return err
}

View File

@ -191,3 +191,49 @@ func GenerateImpersonationRules(subjects []rbacv1.Subject) []rbacv1.PolicyRule {
return rules
}
// CreateRole just try to create the Role.
func CreateRole(client kubeclient.Interface, roleObj *rbacv1.Role) (*rbacv1.Role, error) {
createdObj, err := client.RbacV1().Roles(roleObj.GetNamespace()).Create(context.TODO(), roleObj, metav1.CreateOptions{})
if err != nil {
if apierrors.IsAlreadyExists(err) {
return roleObj, nil
}
return nil, err
}
return createdObj, nil
}
// CreateRoleBinding just try to create the RoleBinding.
func CreateRoleBinding(client kubeclient.Interface, roleBindingObj *rbacv1.RoleBinding) (*rbacv1.RoleBinding, error) {
createdObj, err := client.RbacV1().RoleBindings(roleBindingObj.GetNamespace()).Create(context.TODO(), roleBindingObj, metav1.CreateOptions{})
if err != nil {
if apierrors.IsAlreadyExists(err) {
return roleBindingObj, nil
}
return nil, err
}
return createdObj, nil
}
// DeleteRole just try to delete the Role.
func DeleteRole(client kubeclient.Interface, namespace, name string) error {
err := client.RbacV1().Roles(namespace).Delete(context.TODO(), name, metav1.DeleteOptions{})
if err != nil && !apierrors.IsNotFound(err) {
return err
}
return nil
}
// DeleteRoleBinding just try to delete the RoleBinding.
func DeleteRoleBinding(client kubeclient.Interface, namespace, name string) error {
err := client.RbacV1().RoleBindings(namespace).Delete(context.TODO(), name, metav1.DeleteOptions{})
if err != nil && !apierrors.IsNotFound(err) {
return err
}
return nil
}