- update the IAM policy to ensure the kubelet permision is skipped

- update the PKI to ensure on new clusters the certificate it not created

hack/.packages

@@ -114,7 +114,6 @@ k8s.io/kops/pkg/templates
 k8s.io/kops/pkg/testutils
 k8s.io/kops/pkg/tokens
 k8s.io/kops/pkg/urls
-k8s.io/kops/pkg/util/fs
 k8s.io/kops/pkg/util/stringorslice
 k8s.io/kops/pkg/util/templater
 k8s.io/kops/pkg/validation

nodeup/pkg/model/BUILD.bazel

@@ -48,7 +48,6 @@ go_library(
         "//pkg/pki:go_default_library",
         "//pkg/systemd:go_default_library",
         "//pkg/tokens:go_default_library",
-        "//pkg/util/fs:go_default_library",
         "//upup/pkg/fi:go_default_library",
         "//upup/pkg/fi/nodeup/nodetasks:go_default_library",
         "//upup/pkg/fi/utils:go_default_library",
@@ -62,6 +61,7 @@ go_library(
         "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/intstr:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/authentication/user:go_default_library",
     ],
 )
 

nodeup/pkg/model/context.go

@@ -115,7 +115,19 @@ func (c *NodeupModelContext) CNIBinDir() string {
 
 // KubeletBootstrapConfig is the path the bootstrap config file
 func (c *NodeupModelContext) KubeletBootstrapConfig() string {
-	return c.Cluster.Spec.Kubelet.BootstrapKubeconfig
+	path := c.Cluster.Spec.Kubelet.BootstrapKubeconfig
+
+	if c.IsMaster {
+		if c.Cluster.Spec.MasterKubelet != nil && c.Cluster.Spec.MasterKubelet.BootstrapKubeconfig != "" {
+			path = c.Cluster.Spec.MasterKubelet.BootstrapKubeconfig
+		}
+	}
+
+	if path != "" {
+		return path
+	}
+
+	return "/var/lib/kubelet/bootstrap-kubeconfig"
 }
 
 // KubeletKubeConfig is the path of the kubelet kubeconfig file
@@ -252,7 +264,14 @@ func (c *NodeupModelContext) UsesCNI() bool {
 
 // UseBootstrapTokens checks if we are using bootstrap tokens
 func (c *NodeupModelContext) UseBootstrapTokens() bool {
-	return c.Cluster.Spec.Kubelet.BootstrapKubeconfig != ""
+	if c.Cluster.Spec.Kubelet.BootstrapKubeconfig != "" {
+		return true
+	}
+	if c.Cluster.Spec.MasterKubelet != nil && c.Cluster.Spec.MasterKubelet.BootstrapKubeconfig != "" {
+		return true
+	}
+
+	return false
 }
 
 // UseSecureKubelet checks if the kubelet api should be protected by a client certificate. Note: the settings are
@@ -295,15 +314,15 @@ func (c *NodeupModelContext) KubectlPath() string {
 }
 
 // BuildCertificatePairTask creates the tasks to pull down the certificate and private key
-func (c *NodeupModelContext) BuildCertificatePairTask(ctx *fi.ModelBuilderContext, name, path string) error {
+func (c *NodeupModelContext) BuildCertificatePairTask(ctx *fi.ModelBuilderContext, key, path, filename string) error {
-	certificate := fmt.Sprintf("%s/%s.pem", path, name)
+	certificateName := fmt.Sprintf("%s/%s.pem", strings.TrimSuffix(path, "/"), filename)
-	key := fmt.Sprintf("%s/%s-key.pem", path, name)
+	keyName := fmt.Sprintf("%s/%s-key.pem", strings.TrimSuffix(path, "/"), filename)
 
-	if err := c.BuildCertificateTask(ctx, name, certificate); err != nil {
+	if err := c.BuildCertificateTask(ctx, key, certificateName); err != nil {
 		return err
 	}
 
-	return c.BuildPrivateKeyTask(ctx, name, key)
+	return c.BuildPrivateKeyTask(ctx, key, keyName)
 }
 
 // BuildCertificateTask is responsible for build a certificate request task
@@ -326,7 +345,7 @@ func (c *NodeupModelContext) BuildCertificateTask(ctx *fi.ModelBuilderContext, n
 		Path:     filepath.Join(c.PathSrvKubernetes(), filename),
 		Contents: fi.NewStringResource(serialized),
 		Type:     nodetasks.FileType_File,
-		Mode:     s("0400"),
+		Mode:     s("0600"),
 	})
 
 	return nil
@@ -352,7 +371,7 @@ func (c *NodeupModelContext) BuildPrivateKeyTask(ctx *fi.ModelBuilderContext, na
 		Path:     filepath.Join(c.PathSrvKubernetes(), filename),
 		Contents: fi.NewStringResource(serialized),
 		Type:     nodetasks.FileType_File,
-		Mode:     s("0400"),
+		Mode:     s("0600"),
 	})
 
 	return nil

nodeup/pkg/model/kubelet.go

@@ -17,7 +17,6 @@ limitations under the License.
 package model
 
 import (
-	"context"
 	"crypto/x509"
 	"crypto/x509/pkix"
 	"fmt"
@@ -30,7 +29,6 @@ import (
 	"k8s.io/kops/pkg/flagbuilder"
 	"k8s.io/kops/pkg/pki"
 	"k8s.io/kops/pkg/systemd"
-	"k8s.io/kops/pkg/util/fs"
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kops/upup/pkg/fi/nodeup/nodetasks"
 	"k8s.io/kops/upup/pkg/fi/utils"
@@ -39,6 +37,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/golang/glog"
 	"k8s.io/api/core/v1"
+	"k8s.io/apiserver/pkg/authentication/user"
 )
 
 const (
@@ -93,11 +92,6 @@ func (b *KubeletBuilder) Build(c *fi.ModelBuilderContext) error {
 		if b.UseBootstrapTokens() {
 			glog.V(3).Info("kubelet bootstrap tokens are enabled")
 
-			nodename, err := b.NodeName()
-			if err != nil {
-				return err
-			}
-
 			// @check if a master and if so, we bypass the token strapping and instead generate our own kubeconfig
 			if b.IsMaster {
 				task, err := b.buildMasterKubeletKubeconfig()
@@ -105,21 +99,21 @@ func (b *KubeletBuilder) Build(c *fi.ModelBuilderContext) error {
 					return err
 				}
 				c.AddTask(task)
-			} else {
-				timeout := 5 * time.Minute
 
-				ctx, cancel := context.WithTimeout(context.Background(), timeout)
+				name := "node-authorizer"
-				defer cancel()
+				if err := b.BuildCertificatePairTask(c, name, "node-authorizer/", "tls"); err != nil {
-				// @step: we are a Node, lets wait for the bootstrap file to appear. This is being performed
-				// an external process for now
-				glog.V(3).Infof("node: %s waiting for bootstrap: %s (%s) to be available", nodename, timeout.String(), b.KubeletBootstrapConfig())
-
-				if err := fs.WaitForFile(ctx, b.KubeletBootstrapConfig()); err != nil {
-					glog.Errorf("node: %s has timed out waiting for bootstrap: %s", nodename, b.KubeletBootstrapConfig())
 					return err
 				}
 
-				glog.V(3).Info("kubelet bootstrap configuration is available, continuing")
+			} else {
+				name := "node-authorizer-client"
+				if err := b.BuildCertificatePairTask(c, name, "node-authorizer/", "tls"); err != nil {
+					return err
+				}
+				glog.V(3).Info("kubelet service will wait for bootstrap configuration: %s", b.KubeletBootstrapConfig())
+			}
+			if err := b.BuildCertificateTask(c, fi.CertificateId_CA, "node-authorizer/ca.pem"); err != nil {
+				return err
 			}
 		} else {
 			kubeconfig, err := b.BuildPKIKubeconfig("kubelet")
@@ -235,6 +229,10 @@ func (b *KubeletBuilder) buildSystemdService() *nodetasks.Service {
 	manifest.Set("Unit", "Documentation", "https://github.com/kubernetes/kubernetes")
 	manifest.Set("Unit", "After", "docker.service")
 
+	if b.UseBootstrapTokens() && !b.IsMaster {
+		manifest.Set("Unit", "ConditionPathExists", b.KubeletBootstrapConfig())
+	}
+
 	if b.Distribution == distros.DistributionCoreOS {
 		// We add /opt/kubernetes/bin for our utilities (socat)
 		manifest.Set("Service", "Environment", "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/kubernetes/bin")
@@ -521,7 +519,7 @@ func (b *KubeletBuilder) buildMasterKubeletKubeconfig() (*nodetasks.File, error)
 
 	template.Subject = pkix.Name{
 		CommonName:   fmt.Sprintf("system:node:%s", nodeName),
-		Organization: []string{"system:nodes"},
+		Organization: []string{user.NodesGroup},
 	}
 
 	// https://tools.ietf.org/html/rfc5280#section-4.2.1.3

pkg/model/components/apiserver.go

@@ -92,7 +92,7 @@ func (b *KubeAPIServerOptionsBuilder) BuildOptions(o interface{}) error {
 	} else if clusterSpec.Authorization.RBAC != nil {
 		var modes []string
 
-		if b.IsKubernetesGTE("1.9") {
+		if b.IsKubernetesGTE("1.10") {
 			// Enable the Node authorizer, used for special per-node RBAC policies
 			modes = append(modes, "Node")
 		}

pkg/model/context.go

@@ -249,7 +249,14 @@ func (m *KopsModelContext) CloudTags(name string, shared bool) map[string]string
 
 // UseBootstrapTokens checks if bootstrap tokens are enabled
 func (m *KopsModelContext) UseBootstrapTokens() bool {
-	return m.Cluster.Spec.Kubelet.BootstrapKubeconfig != ""
+	if m.Cluster.Spec.Kubelet.BootstrapKubeconfig != "" {
+		return true
+	}
+	if m.Cluster.Spec.MasterKubelet != nil && m.Cluster.Spec.MasterKubelet.BootstrapKubeconfig != "" {
+		return true
+	}
+
+	return false
 }
 
 // UsesBastionDns checks if we should use a specific name for the bastion dns

pkg/model/iam/iam_builder.go

@@ -348,12 +348,30 @@ func (b *PolicyBuilder) AddS3Permissions(p *Policy) (*Policy, error) {
 							strings.Join([]string{b.IAMPrefix(), ":s3:::", iamS3Path, "/instancegroup/*"}, ""),
 							strings.Join([]string{b.IAMPrefix(), ":s3:::", iamS3Path, "/pki/issued/*"}, ""),
 							strings.Join([]string{b.IAMPrefix(), ":s3:::", iamS3Path, "/pki/private/kube-proxy/*"}, ""),
-							strings.Join([]string{b.IAMPrefix(), ":s3:::", iamS3Path, "/pki/private/kubelet/*"}, ""),
 							strings.Join([]string{b.IAMPrefix(), ":s3:::", iamS3Path, "/pki/ssh/*"}, ""),
 							strings.Join([]string{b.IAMPrefix(), ":s3:::", iamS3Path, "/secrets/dockerconfig"}, ""),
 						),
 					})
 
+					// @check if bootstrap tokens are enabled and if so, we disable access for the nodes
+					if !b.UseBootstrapTokens() {
+						p.Statement = append(p.Statement, &Statement{
+							Effect: StatementEffectAllow,
+							Action: stringorslice.Slice([]string{"s3:Get*"}),
+							Resource: stringorslice.Of(
+								strings.Join([]string{b.IAMPrefix(), ":s3:::", iamS3Path, "/pki/private/kubelet/*"}, ""),
+							),
+						})
+					} else {
+						p.Statement = append(p.Statement, &Statement{
+							Effect: StatementEffectAllow,
+							Action: stringorslice.Slice([]string{"s3:Get*"}),
+							Resource: stringorslice.Of(
+								strings.Join([]string{b.IAMPrefix(), ":s3:::", iamS3Path, "/pki/private/node-authorizer-client/*"}, ""),
+							),
+						})
+					}
+
 					if b.Cluster.Spec.Networking != nil {
 						// @check if kuberoute is enabled and permit access to the private key
 						if b.Cluster.Spec.Networking.Kuberouter != nil {
@@ -469,6 +487,19 @@ func (b *PolicyResource) Open() (io.Reader, error) {
 	return bytes.NewReader([]byte(j)), nil
 }
 
+// UseBootstrapTokens check if we are using bootstrap tokens - @TODO, i don't like this we should probably pass in
+// the kops model into the builder rather than duplicating the code. I'll leave for anothe PR
+func (b *PolicyBuilder) UseBootstrapTokens() bool {
+	if b.Cluster.Spec.Kubelet != nil && b.Cluster.Spec.Kubelet.BootstrapKubeconfig != "" {
+		return true
+	}
+	if b.Cluster.Spec.MasterKubelet != nil && b.Cluster.Spec.MasterKubelet.BootstrapKubeconfig != "" {
+		return true
+	}
+
+	return false
+}
+
 func addECRPermissions(p *Policy) {
 	// TODO - I think we can just have GetAuthorizationToken here, as we are not
 	// TODO - making any API calls except for GetAuthorizationToken.
@@ -755,59 +786,59 @@ func addRomanaCNIPermissions(p *Policy, resource stringorslice.StringOrSlice, le
 	if legacyIAM {
 		// Legacy IAM provides ec2:*, so no additional permissions required
 		return
-	} else {
+	}
-		// Romana requires additional Describe permissions
+
-		// Comments are which Romana component makes the call
+	// Romana requires additional Describe permissions
-		p.Statement = append(p.Statement,
+	// Comments are which Romana component makes the call
-			&Statement{
+	p.Statement = append(p.Statement,
-				Effect: StatementEffectAllow,
+		&Statement{
-				Action: stringorslice.Slice([]string{
+			Effect: StatementEffectAllow,
-					"ec2:DescribeAvailabilityZones", // vpcrouter
+			Action: stringorslice.Slice([]string{
-					"ec2:DescribeVpcs",              // vpcrouter
+				"ec2:DescribeAvailabilityZones", // vpcrouter
-				}),
+				"ec2:DescribeVpcs",              // vpcrouter
-				Resource: resource,
+			}),
-			},
+			Resource: resource,
-			&Statement{
+		},
-				Effect: StatementEffectAllow,
+		&Statement{
-				Action: stringorslice.Slice([]string{
+			Effect: StatementEffectAllow,
-					"ec2:CreateRoute",  // vpcrouter
+			Action: stringorslice.Slice([]string{
-					"ec2:DeleteRoute",  // vpcrouter
+				"ec2:CreateRoute",  // vpcrouter
-					"ec2:ReplaceRoute", // vpcrouter
+				"ec2:DeleteRoute",  // vpcrouter
-				}),
+				"ec2:ReplaceRoute", // vpcrouter
-				Resource: resource,
+			}),
-				Condition: Condition{
+			Resource: resource,
-					"StringEquals": map[string]string{
+			Condition: Condition{
-						"ec2:ResourceTag/KubernetesCluster": clusterName,
+				"StringEquals": map[string]string{
-					},
+					"ec2:ResourceTag/KubernetesCluster": clusterName,
 				},
 			},
-		)
+		},
-	}
+	)
 }
 
 func addAmazonVPCCNIPermissions(p *Policy, resource stringorslice.StringOrSlice, legacyIAM bool, clusterName string) {
 	if legacyIAM {
 		// Legacy IAM provides ec2:*, so no additional permissions required
 		return
-	} else {
-		p.Statement = append(p.Statement,
-			&Statement{
-				Effect: StatementEffectAllow,
-				Action: stringorslice.Slice([]string{
-					"ec2:CreateNetworkInterface",
-					"ec2:AttachNetworkInterface",
-					"ec2:DeleteNetworkInterface",
-					"ec2:DetachNetworkInterface",
-					"ec2:DescribeNetworkInterfaces",
-					"ec2:DescribeInstances",
-					"ec2:ModifyNetworkInterfaceAttribute",
-					"ec2:AssignPrivateIpAddresses",
-					"tag:TagResources",
-				}),
-				Resource: resource,
-			},
-		)
 	}
+
+	p.Statement = append(p.Statement,
+		&Statement{
+			Effect: StatementEffectAllow,
+			Action: stringorslice.Slice([]string{
+				"ec2:CreateNetworkInterface",
+				"ec2:AttachNetworkInterface",
+				"ec2:DeleteNetworkInterface",
+				"ec2:DetachNetworkInterface",
+				"ec2:DescribeNetworkInterfaces",
+				"ec2:DescribeInstances",
+				"ec2:ModifyNetworkInterfaceAttribute",
+				"ec2:AssignPrivateIpAddresses",
+				"tag:TagResources",
+			}),
+			Resource: resource,
+		},
+	)
 }
 
 func createResource(b *PolicyBuilder) stringorslice.StringOrSlice {

pkg/model/iam/tests/iam_builder_node_strict.json

@@ -33,10 +33,16 @@
         "arn:aws:s3:::kops-tests/iam-builder-test.k8s.local/instancegroup/*",
         "arn:aws:s3:::kops-tests/iam-builder-test.k8s.local/pki/issued/*",
         "arn:aws:s3:::kops-tests/iam-builder-test.k8s.local/pki/private/kube-proxy/*",
-        "arn:aws:s3:::kops-tests/iam-builder-test.k8s.local/pki/private/kubelet/*",
         "arn:aws:s3:::kops-tests/iam-builder-test.k8s.local/pki/ssh/*",
         "arn:aws:s3:::kops-tests/iam-builder-test.k8s.local/secrets/dockerconfig"
       ]
+    },
+    {
+      "Effect": "Allow",
+      "Action": [
+        "s3:Get*"
+      ],
+      "Resource": "arn:aws:s3:::kops-tests/iam-builder-test.k8s.local/pki/private/kubelet/*"
     }
   ]
 }

pkg/model/iam/tests/iam_builder_node_strict_ecr.json

@@ -33,11 +33,17 @@
         "arn:aws:s3:::kops-tests/iam-builder-test.k8s.local/instancegroup/*",
         "arn:aws:s3:::kops-tests/iam-builder-test.k8s.local/pki/issued/*",
         "arn:aws:s3:::kops-tests/iam-builder-test.k8s.local/pki/private/kube-proxy/*",
-        "arn:aws:s3:::kops-tests/iam-builder-test.k8s.local/pki/private/kubelet/*",
         "arn:aws:s3:::kops-tests/iam-builder-test.k8s.local/pki/ssh/*",
         "arn:aws:s3:::kops-tests/iam-builder-test.k8s.local/secrets/dockerconfig"
       ]
     },
+    {
+      "Effect": "Allow",
+      "Action": [
+        "s3:Get*"
+      ],
+      "Resource": "arn:aws:s3:::kops-tests/iam-builder-test.k8s.local/pki/private/kubelet/*"
+    },
     {
       "Effect": "Allow",
       "Action": [

pkg/model/pki.go

@@ -18,6 +18,7 @@ package model
 
 import (
 	"fmt"
+	"strings"
 
 	"k8s.io/kops/pkg/tokens"
 	"k8s.io/kops/upup/pkg/fi"
@@ -53,17 +54,18 @@ func (b *PKIModelBuilder) Build(c *fi.ModelBuilderContext) error {
 	c.AddTask(defaultCA)
 
 	{
-
+		// @check of bootstrap tokens are enable if so, disable the creation of the kubelet certificate - we also
-		t := &fitasks.Keypair{
+		// block at the IAM level for AWS cluster for pre-existing clusters.
-			Name:      fi.String("kubelet"),
+		if !b.UseBootstrapTokens() {
-			Lifecycle: b.Lifecycle,
+			c.AddTask(&fitasks.Keypair{
-
+				Name:      fi.String("kubelet"),
-			Subject: "o=" + user.NodesGroup + ",cn=kubelet",
+				Lifecycle: b.Lifecycle,
-			Type:    "client",
+				Subject:   "o=" + user.NodesGroup + ",cn=kubelet",
-			Signer:  defaultCA,
+				Type:      "client",
-			Format:  format,
+				Signer:    defaultCA,
+				Format:    format,
+			})
 		}
-		c.AddTask(t)
 	}
 	{
 		// Generate a kubelet client certificate for api to speak securely to kubelets. This change was first
@@ -284,17 +286,20 @@ func (b *PKIModelBuilder) Build(c *fi.ModelBuilderContext) error {
 	// But I'm conscious not to do too much work on bootstrap tokens as it might overlay further down the
 	// line with the machines api
 	if b.UseBootstrapTokens() {
+		serviceName := "node-authorizer-internal"
+
 		alternateNames := []string{
 			"127.0.0.1",
 			"localhost",
-			"node-bootstrap-internal",
+			serviceName,
-			"node-bootstrap-internal." + b.Cluster.Spec.DNSZone,
+			strings.Join([]string{serviceName, b.Cluster.Name}, "."),
+			strings.Join([]string{serviceName, b.Cluster.Spec.DNSZone}, "."),
 		}
 
 		// @note: the certificate used by the node authorizers
 		c.AddTask(&fitasks.Keypair{
-			Name:           fi.String("node-bootstrap"),
+			Name:           fi.String("node-authorizer"),
-			Subject:        "cn=node-bootstrap",
+			Subject:        "cn=node-authorizaer",
 			Type:           "server",
 			AlternateNames: alternateNames,
 			Signer:         defaultCA,
@@ -303,8 +308,8 @@ func (b *PKIModelBuilder) Build(c *fi.ModelBuilderContext) error {
 
 		// @note: we use this for mutual tls between between node and authorizer
 		c.AddTask(&fitasks.Keypair{
-			Name:    fi.String("node-bootstrap-client"),
+			Name:    fi.String("node-authorizer-client"),
-			Subject: "cn=node-bootstrap-client",
+			Subject: "cn=node-authorizer-client",
 			Type:    "client",
 			Signer:  defaultCA,
 			Format:  format,

pkg/util/fs/BUILD.bazel

@@ -1,8 +0,0 @@
-load("@io_bazel_rules_go//go:def.bzl", "go_library")
-
-go_library(
-    name = "go_default_library",
-    srcs = ["wait.go"],
-    importpath = "k8s.io/kops/pkg/util/fs",
-    visibility = ["//visibility:public"],
-)

pkg/util/fs/wait.go

@@ -1,71 +0,0 @@
-/*
-Copyright 2018 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package fs
-
-import (
-	"context"
-	"errors"
-	"os"
-	"time"
-)
-
-// ErrTimeout indicates the operation has timed out
-var ErrTimeout = errors.New("operation timeout")
-
-// WaitForFile is responsible for waiting for file to appear or timeout
-func WaitForFile(ctx context.Context, path string) error {
-	doneCh := make(chan struct{}, 0)
-
-	// @step: we wait for the bootstrap token file to appear
-	go func() {
-		ticker := time.NewTicker(5 * time.Second)
-		defer ticker.Stop()
-
-		for {
-			select {
-			case <-ctx.Done():
-				return
-			case <-ticker.C:
-				if found, _ := FileExists(path); found {
-					doneCh <- struct{}{}
-					return
-				}
-			}
-		}
-	}()
-
-	select {
-	case <-ctx.Done():
-		return ctx.Err()
-	case <-doneCh:
-	}
-
-	return nil
-}
-
-// FileExists checks if a file exists
-func FileExists(path string) (bool, error) {
-	if _, err := os.Stat(path); err != nil {
-		if os.IsNotExist(err) {
-			return false, nil
-		}
-
-		return false, err
-	}
-
-	return true, nil
-}

upup/models/cloudup/resources/addons/podsecuritypolicy.addons.k8s.io/k8s-1.10.yaml.template

@@ -54,12 +54,10 @@ subjects:
 - kind: Group
   name: system:masters
   apiGroup: rbac.authorization.k8s.io
-{{- if not UseBootstrapTokens }}
 # permit the kubelets to access this policy (used for manifests)
 - kind: User
   name: kubelet
   apiGroup: rbac.authorization.k8s.io
-{{- end }}
 ---
 kind: RoleBinding
 apiVersion: rbac.authorization.k8s.io/v1beta1