From 59c17d911f764f037ce483815ec3bbfd27a9e9e7 Mon Sep 17 00:00:00 2001 From: lonelyCZ <531187475@qq.com> Date: Mon, 22 Aug 2022 15:42:48 +0800 Subject: [PATCH] Set the required RBAC to enable the certificate signing request Signed-off-by: lonelyCZ <531187475@qq.com> --- .../bootstraptoken/agent/tlsbootstrap.go | 70 +++++++++++++++ pkg/karmadactl/cmdinit/karmada/deploy.go | 18 ++++ pkg/karmadactl/cmdinit/karmada/rbac.go | 88 +++++++++++++++++++ .../cmdinit/kubernetes/deployments.go | 3 +- 4 files changed, 177 insertions(+), 2 deletions(-) create mode 100644 pkg/karmadactl/cmdinit/bootstraptoken/agent/tlsbootstrap.go diff --git a/pkg/karmadactl/cmdinit/bootstraptoken/agent/tlsbootstrap.go b/pkg/karmadactl/cmdinit/bootstraptoken/agent/tlsbootstrap.go new file mode 100644 index 000000000..f181dec70 --- /dev/null +++ b/pkg/karmadactl/cmdinit/bootstraptoken/agent/tlsbootstrap.go @@ -0,0 +1,70 @@ +package agent + +import ( + rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/klog/v2" + + "github.com/karmada-io/karmada/pkg/karmadactl/cmdinit/utils" +) + +const ( + // KarmadaAgentBootstrapperClusterRoleName defines the name of the auto-bootstrapped ClusterRole for letting someone post a CSR + KarmadaAgentBootstrapperClusterRoleName = "system:node-bootstrapper" + // KarmadaAgentBootstrap defines the name of the ClusterRoleBinding that lets Karmada Agent post CSRs + KarmadaAgentBootstrap = "karmada:agent-bootstrap" + // KarmadaAgentGroup defines the group of Karmada Agent + KarmadaAgentGroup = "system:nodes" + // KarmadaAgentAutoApproveBootstrapClusterRoleBinding defines the name of the ClusterRoleBinding that makes the csrapprover approve agent CSRs + KarmadaAgentAutoApproveBootstrapClusterRoleBinding = "karmada:agent-autoapprove-bootstrap" + // KarmadaAgentAutoApproveCertificateRotationClusterRoleBinding defines name of the ClusterRoleBinding that makes the csrapprover approve agent auto rotated CSRs + KarmadaAgentAutoApproveCertificateRotationClusterRoleBinding = "karmada:agent-autoapprove-certificate-rotation" + // CSRAutoApprovalClusterRoleName defines the name of the auto-bootstrapped ClusterRole for making the csrapprover controller auto-approve the CSR + CSRAutoApprovalClusterRoleName = "system:certificates.k8s.io:certificatesigningrequests:nodeclient" + // KarmadaAgentSelfCSRAutoApprovalClusterRoleName is a role for automatic CSR approvals for automatically rotated agent certificates + KarmadaAgentSelfCSRAutoApprovalClusterRoleName = "system:certificates.k8s.io:certificatesigningrequests:selfnodeclient" + // KarmadaAgentBootstrapTokenAuthGroup specifies which group a Karmada Agent Bootstrap Token should be authenticated in + KarmadaAgentBootstrapTokenAuthGroup = "system:bootstrappers:karmada:default-cluster-token" +) + +// AllowBootstrapTokensToPostCSRs creates RBAC rules in a way the makes Karmada Agent Bootstrap Tokens able to post CSRs +func AllowBootstrapTokensToPostCSRs(clientSet *kubernetes.Clientset) error { + klog.Infoln("[bootstrap-token] configured RBAC rules to allow Karmada Agent Bootstrap tokens to post CSRs in order for agent to get long term certificate credentials") + + clusterRoleBinding := utils.ClusterRoleBindingFromSubjects(KarmadaAgentBootstrap, KarmadaAgentBootstrapperClusterRoleName, + []rbacv1.Subject{ + { + Kind: rbacv1.GroupKind, + Name: KarmadaAgentBootstrapTokenAuthGroup, + }, + }, nil) + return utils.CreateIfNotExistClusterRoleBinding(clientSet, clusterRoleBinding) +} + +// AutoApproveKarmadaAgentBootstrapTokens creates RBAC rules in a way that makes Karmada Agent Bootstrap Tokens' CSR auto-approved by the csrapprover controller +func AutoApproveKarmadaAgentBootstrapTokens(clientSet *kubernetes.Clientset) error { + klog.Infoln("[bootstrap-token] configured RBAC rules to allow the csrapprover controller automatically approve CSRs from a Karmada Agent Bootstrap Token") + + clusterRoleBinding := utils.ClusterRoleBindingFromSubjects(KarmadaAgentAutoApproveBootstrapClusterRoleBinding, CSRAutoApprovalClusterRoleName, + []rbacv1.Subject{ + { + Kind: rbacv1.GroupKind, + Name: KarmadaAgentBootstrapTokenAuthGroup, + }, + }, nil) + return utils.CreateIfNotExistClusterRoleBinding(clientSet, clusterRoleBinding) +} + +// AutoApproveAgentCertificateRotation creates RBAC rules in a way that makes Agent certificate rotation CSR auto-approved by the csrapprover controller +func AutoApproveAgentCertificateRotation(clientSet *kubernetes.Clientset) error { + klog.Infoln("[bootstrap-token] configured RBAC rules to allow certificate rotation for all agent client certificates in the member cluster") + + clusterRoleBinding := utils.ClusterRoleBindingFromSubjects(KarmadaAgentAutoApproveCertificateRotationClusterRoleBinding, KarmadaAgentSelfCSRAutoApprovalClusterRoleName, + []rbacv1.Subject{ + { + Kind: rbacv1.GroupKind, + Name: KarmadaAgentGroup, + }, + }, nil) + return utils.CreateIfNotExistClusterRoleBinding(clientSet, clusterRoleBinding) +} diff --git a/pkg/karmadactl/cmdinit/karmada/deploy.go b/pkg/karmadactl/cmdinit/karmada/deploy.go index 0fdc0469f..ec8737480 100644 --- a/pkg/karmadactl/cmdinit/karmada/deploy.go +++ b/pkg/karmadactl/cmdinit/karmada/deploy.go @@ -22,6 +22,7 @@ import ( apiregistrationv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1" "sigs.k8s.io/yaml" + bootstrapagent "github.com/karmada-io/karmada/pkg/karmadactl/cmdinit/bootstraptoken/agent" "github.com/karmada-io/karmada/pkg/karmadactl/cmdinit/bootstraptoken/clusterinfo" "github.com/karmada-io/karmada/pkg/karmadactl/cmdinit/options" "github.com/karmada-io/karmada/pkg/karmadactl/cmdinit/utils" @@ -121,6 +122,23 @@ func createExtralResources(clientSet *kubernetes.Clientset, dir string) error { return fmt.Errorf("error creating clusterinfo RBAC rules: %v", err) } + // grant limited access permission to 'karmada-agent' + if err := grantAccessPermissionToAgent(clientSet); err != nil { + return err + } + + if err := bootstrapagent.AllowBootstrapTokensToPostCSRs(clientSet); err != nil { + return err + } + + if err := bootstrapagent.AutoApproveKarmadaAgentBootstrapTokens(clientSet); err != nil { + return err + } + + if err := bootstrapagent.AutoApproveAgentCertificateRotation(clientSet); err != nil { + return err + } + return nil } diff --git a/pkg/karmadactl/cmdinit/karmada/rbac.go b/pkg/karmadactl/cmdinit/karmada/rbac.go index 0a4457c6d..03334cb38 100644 --- a/pkg/karmadactl/cmdinit/karmada/rbac.go +++ b/pkg/karmadactl/cmdinit/karmada/rbac.go @@ -7,6 +7,11 @@ import ( "github.com/karmada-io/karmada/pkg/karmadactl/cmdinit/utils" ) +const ( + karmadaAgentAccessClusterRole = "system:karmada:agent" + karmadaAgentGroup = "system:nodes" +) + // grantProxyPermissionToAdmin grants the proxy permission to "system:admin" func grantProxyPermissionToAdmin(clientSet *kubernetes.Clientset) error { proxyAdminClusterRole := utils.ClusterRoleFromRules(clusterProxyAdminRole, []rbacv1.PolicyRule{ @@ -34,3 +39,86 @@ func grantProxyPermissionToAdmin(clientSet *kubernetes.Clientset) error { return nil } + +// grantAccessPermissionToAgent grants the limited access permmission to 'karmada-agent' +func grantAccessPermissionToAgent(clientSet *kubernetes.Clientset) error { + clusterRole := utils.ClusterRoleFromRules(karmadaAgentAccessClusterRole, []rbacv1.PolicyRule{ + { + APIGroups: []string{"authentication.k8s.io"}, + Resources: []string{"tokenreviews"}, + Verbs: []string{"create"}, + }, + { + APIGroups: []string{"cluster.karmada.io"}, + Resources: []string{"clusters"}, + Verbs: []string{"create", "get", "list", "watch", "patch", "update"}, + }, + { + APIGroups: []string{"cluster.karmada.io"}, + Resources: []string{"clusters/status"}, + Verbs: []string{"patch", "update"}, + }, + { + APIGroups: []string{"work.karmada.io"}, + Resources: []string{"works"}, + Verbs: []string{"get", "list", "watch"}, + }, + { + APIGroups: []string{"work.karmada.io"}, + Resources: []string{"works/status"}, + Verbs: []string{"patch", "update"}, + }, + { + APIGroups: []string{"config.karmada.io"}, + Resources: []string{"resourceinterpreterwebhookconfigurations"}, + Verbs: []string{"get", "list", "watch"}, + }, + { + APIGroups: []string{"config.karmada.io"}, + Resources: []string{"resourceinterpreterwebhookconfigurations"}, + Verbs: []string{"get", "list", "watch"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"namespaces"}, + Verbs: []string{"get", "list", "watch", "create"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"secrets"}, + Verbs: []string{"get", "list", "watch", "create", "patch"}, + }, + { + APIGroups: []string{"coordination.k8s.io"}, + Resources: []string{"leases"}, + Verbs: []string{"create", "delete", "get", "patch", "update"}, + }, + { + APIGroups: []string{"certificates.k8s.io"}, + Resources: []string{"certificatesigningrequests"}, + Verbs: []string{"create", "get", "list", "watch"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"events"}, + Verbs: []string{"create", "patch", "update"}, + }, + }, nil) + err := utils.CreateIfNotExistClusterRole(clientSet, clusterRole) + if err != nil { + return err + } + + clusterRoleBinding := utils.ClusterRoleBindingFromSubjects(karmadaAgentAccessClusterRole, karmadaAgentAccessClusterRole, + []rbacv1.Subject{ + { + Kind: rbacv1.GroupKind, + Name: karmadaAgentGroup, + }}, nil) + err = utils.CreateIfNotExistClusterRoleBinding(clientSet, clusterRoleBinding) + if err != nil { + return err + } + + return nil +} diff --git a/pkg/karmadactl/cmdinit/kubernetes/deployments.go b/pkg/karmadactl/cmdinit/kubernetes/deployments.go index 7fe89d3c6..fee7ef32a 100644 --- a/pkg/karmadactl/cmdinit/kubernetes/deployments.go +++ b/pkg/karmadactl/cmdinit/kubernetes/deployments.go @@ -65,7 +65,6 @@ func (i *CommandInitOption) karmadaAPIServerContainerCommand() []string { "--allow-privileged=true", "--authorization-mode=Node,RBAC", fmt.Sprintf("--client-ca-file=%s/%s.crt", karmadaCertsVolumeMountPath, options.CaCertAndKeyName), - "--enable-admission-plugins=NodeRestriction", "--enable-bootstrap-token-auth=true", fmt.Sprintf("--etcd-cafile=%s/%s.crt", karmadaCertsVolumeMountPath, options.EtcdCaCertAndKeyName), fmt.Sprintf("--etcd-certfile=%s/%s.crt", karmadaCertsVolumeMountPath, options.EtcdClientCertAndKeyName), @@ -268,7 +267,7 @@ func (i *CommandInitOption) makeKarmadaKubeControllerManagerDeployment() *appsv1 fmt.Sprintf("--cluster-name=%s", options.ClusterName), fmt.Sprintf("--cluster-signing-cert-file=%s/%s.crt", karmadaCertsVolumeMountPath, options.CaCertAndKeyName), fmt.Sprintf("--cluster-signing-key-file=%s/%s.key", karmadaCertsVolumeMountPath, options.CaCertAndKeyName), - "--controllers=namespace,garbagecollector,serviceaccount-token", + "--controllers=namespace,garbagecollector,serviceaccount-token,ttl-after-finished,bootstrapsigner,tokencleaner,csrapproving,csrcleaner,csrsigning", "--kubeconfig=/etc/kubeconfig", "--leader-elect=true", fmt.Sprintf("--leader-elect-resource-namespace=%s", i.Namespace),