Merge pull request #9704 from nckturner/aws-cloud-controller

Add aws-cloud-controller-manager config to addons
2020-12-02 12:17:00 -08:00 · 2020-12-02 12:17:00 -08:00 · 443567426e
parent 091f066543 c9feb36f3f
commit 443567426e
13 changed files with 704 additions and 0 deletions
--- a/k8s/crds/kops.k8s.io_clusters.yaml
+++ b/k8s/crds/kops.k8s.io_clusters.yaml
@ -1231,6 +1231,9 @@ spec:
                  experimentalClusterSigningDuration:
                    description: ExperimentalClusterSigningDuration is the duration that determines the length of duration that the signed certificates will be given. (default 8760h0m0s)
                    type: string
+                  externalCloudVolumePlugin:
+                    description: ExternalCloudVolumePlugin is a fallback mechanism that allows a legacy, in-tree cloudprovider to be used for volume plugins even when an external cloud controller manager is being used.  This can be used instead of installing CSI.  The value should be the same as is used for the --cloud-provider flag, i.e. "aws".
+                    type: string
                  featureGates:
                    additionalProperties:
                      type: string
--- a/pkg/apis/kops/componentconfig.go
+++ b/pkg/apis/kops/componentconfig.go
@ -613,6 +613,10 @@ type KubeControllerManagerConfig struct {
 	AuthorizationKubeconfig string `json:"authorizationKubeconfig,omitempty" flag:"authorization-kubeconfig"`
 	// AuthorizationAlwaysAllowPaths is the list of HTTP paths to skip during authorization
 	AuthorizationAlwaysAllowPaths []string `json:"authorizationAlwaysAllowPaths,omitempty" flag:"authorization-always-allow-paths"`
+	// ExternalCloudVolumePlugin is a fallback mechanism that allows a legacy, in-tree cloudprovider to be used for volume plugins
+	// even when an external cloud controller manager is being used.  This can be used instead of installing CSI.  The value should
+	// be the same as is used for the --cloud-provider flag, i.e. "aws".
+	ExternalCloudVolumePlugin string `json:"externalCloudVolumePlugin,omitempty" flag:"external-cloud-volume-plugin"`

 	// EnableProfiling enables profiling via web interface host:port/debug/pprof/
 	EnableProfiling *bool `json:"enableProfiling,omitempty" flag:"profiling"`
--- a/pkg/apis/kops/v1alpha2/componentconfig.go
+++ b/pkg/apis/kops/v1alpha2/componentconfig.go
@ -614,6 +614,8 @@ type KubeControllerManagerConfig struct {
 	AuthorizationKubeconfig string `json:"authorizationKubeconfig,omitempty" flag:"authorization-kubeconfig"`
 	// AuthorizationAlwaysAllowPaths is the list of HTTP paths to skip during authorization
 	AuthorizationAlwaysAllowPaths []string `json:"authorizationAlwaysAllowPaths,omitempty" flag:"authorization-always-allow-paths"`
+	// ExternalCloudVolumePlugin is a fallback mechanism that allows a legacy, in-tree cloudprovider to be used for volume plugins even when an external cloud controller manager is being used.  This can be used instead of installing CSI.  The value should be the same as is used for the --cloud-provider flag, i.e. "aws".
+	ExternalCloudVolumePlugin string `json:"externalCloudVolumePlugin,omitempty" flag:"external-cloud-volume-plugin"`

 	// EnableProfiling enables profiling via web interface host:port/debug/pprof/
 	EnableProfiling *bool `json:"enableProfiling,omitempty" flag:"profiling"`
--- a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go
+++ b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go
@ -4170,6 +4170,7 @@ func autoConvert_v1alpha2_KubeControllerManagerConfig_To_kops_KubeControllerMana
 	out.AuthenticationKubeconfig = in.AuthenticationKubeconfig
 	out.AuthorizationKubeconfig = in.AuthorizationKubeconfig
 	out.AuthorizationAlwaysAllowPaths = in.AuthorizationAlwaysAllowPaths
+	out.ExternalCloudVolumePlugin = in.ExternalCloudVolumePlugin
 	out.EnableProfiling = in.EnableProfiling
 	return nil
 }
@ -4235,6 +4236,7 @@ func autoConvert_kops_KubeControllerManagerConfig_To_v1alpha2_KubeControllerMana
 	out.AuthenticationKubeconfig = in.AuthenticationKubeconfig
 	out.AuthorizationKubeconfig = in.AuthorizationKubeconfig
 	out.AuthorizationAlwaysAllowPaths = in.AuthorizationAlwaysAllowPaths
+	out.ExternalCloudVolumePlugin = in.ExternalCloudVolumePlugin
 	out.EnableProfiling = in.EnableProfiling
 	return nil
 }
--- a/pkg/model/components/kubecontrollermanager.go
+++ b/pkg/model/components/kubecontrollermanager.go
@ -98,6 +98,14 @@ func (b *KubeControllerManagerOptionsBuilder) BuildOptions(o interface{}) error

 	if clusterSpec.ExternalCloudControllerManager != nil {
 		kcm.CloudProvider = "external"
+
+		// External cloud provider disables KCM volume controllers, so
+		// most users would want to either install CSI or pass
+		// --external-cloud-volume-plugin to the KCM, which runs the
+		// KCM volume controllers.
+		if kcm.ExternalCloudVolumePlugin == "" {
+			klog.Infof("An external cloud controller manager is configured, but ExternalCloudVolumePlugin is not configured for the KCM.  This means a CSI plugin must be installed by the user or else volume management might not work.")
+		}
 	}

 	kcm.LogLevel = 2
--- a/upup/models/bindata.go
+++ b/upup/models/bindata.go
@ -4,6 +4,7 @@
 // upup/models/cloudup/resources/addons/anonymous-issuer-discovery.addons.k8s.io/k8s-1.16.yaml.template
 // upup/models/cloudup/resources/addons/authentication.aws/k8s-1.12.yaml.template
 // upup/models/cloudup/resources/addons/authentication.kope.io/k8s-1.12.yaml
+// upup/models/cloudup/resources/addons/aws-cloud-controller.addons.k8s.io/k8s-1.18.yaml.template
 // upup/models/cloudup/resources/addons/cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml.template
 // upup/models/cloudup/resources/addons/core.addons.k8s.io/addon.yaml
 // upup/models/cloudup/resources/addons/core.addons.k8s.io/k8s-1.12.yaml.template
@ -573,6 +574,185 @@ func cloudupResourcesAddonsAuthenticationKopeIoK8s112Yaml() (*asset, error) {
 	return a, nil
 }

+var _cloudupResourcesAddonsAwsCloudControllerAddonsK8sIoK8s118YamlTemplate = []byte(`---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: aws-cloud-controller-manager
+  namespace: kube-system
+  labels:
+    k8s-app: aws-cloud-controller-manager
+spec:
+  selector:
+    matchLabels:
+      k8s-app: aws-cloud-controller-manager
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        k8s-app: aws-cloud-controller-manager
+    spec:
+      nodeSelector:
+        node-role.kubernetes.io/master: ""
+      tolerations:
+      - key: node.cloudprovider.kubernetes.io/uninitialized
+        value: "true"
+        effect: NoSchedule
+      - key: node-role.kubernetes.io/master
+        effect: NoSchedule
+      serviceAccountName: cloud-controller-manager
+      containers:
+        - name: aws-cloud-controller-manager
+          image: {{ if .ExternalCloudControllerManager.Image }}{{ .ExternalCloudControllerManager.Image }}{{ else }}gcr.io/k8s-staging-provider-aws/cloud-controller-manager:{{AWSCCMTag}}{{ end }}
+          args:
+{{- range $arg := CloudControllerConfigArgv }}
+            - {{ $arg }}
+{{- end }}
+          resources:
+            requests:
+              cpu: 200m
+      hostNetwork: true
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: cloud-controller-manager
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: cloud-controller-manager:apiserver-authentication-reader
+  namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: extension-apiserver-authentication-reader
+subjects:
+- apiGroup: ""
+  kind: ServiceAccount
+  name: cloud-controller-manager
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: system:cloud-controller-manager
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - create
+  - patch
+  - update
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - '*'
+- apiGroups:
+  - ""
+  resources:
+  - nodes/status
+  verbs:
+  - patch
+- apiGroups:
+  - ""
+  resources:
+  - services
+  verbs:
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - services/status
+  verbs:
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - serviceaccounts
+  verbs:
+  - create
+  - get
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumes
+  verbs:
+  - get
+  - list
+  - update
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - endpoints
+  verbs:
+  - create
+  - get
+  - list
+  - watch
+  - update
+- apiGroups:
+  - coordination.k8s.io
+  resources:
+  - leases
+  verbs:
+  - create
+  - get
+  - list
+  - watch
+  - update
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
+  - list
+  - watch
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: system:cloud-controller-manager
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:cloud-controller-manager
+subjects:
+- apiGroup: ""
+  kind: ServiceAccount
+  name: cloud-controller-manager
+  namespace: kube-system
+
+`)
+
+func cloudupResourcesAddonsAwsCloudControllerAddonsK8sIoK8s118YamlTemplateBytes() ([]byte, error) {
+	return _cloudupResourcesAddonsAwsCloudControllerAddonsK8sIoK8s118YamlTemplate, nil
+}
+
+func cloudupResourcesAddonsAwsCloudControllerAddonsK8sIoK8s118YamlTemplate() (*asset, error) {
+	bytes, err := cloudupResourcesAddonsAwsCloudControllerAddonsK8sIoK8s118YamlTemplateBytes()
+	if err != nil {
+		return nil, err
+	}
+
+	info := bindataFileInfo{name: "cloudup/resources/addons/aws-cloud-controller.addons.k8s.io/k8s-1.18.yaml.template", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)}
+	a := &asset{bytes: bytes, info: info}
+	return a, nil
+}
+
 var _cloudupResourcesAddonsClusterAutoscalerAddonsK8sIoK8s115YamlTemplate = []byte(`{{ with .ClusterAutoscaler }}
 # Sourced from https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler/cloudprovider/aws/examples
 ---
@ -15954,6 +16134,7 @@ var _bindata = map[string]func() (*asset, error){
 	"cloudup/resources/addons/anonymous-issuer-discovery.addons.k8s.io/k8s-1.16.yaml.template":            cloudupResourcesAddonsAnonymousIssuerDiscoveryAddonsK8sIoK8s116YamlTemplate,
 	"cloudup/resources/addons/authentication.aws/k8s-1.12.yaml.template":                                  cloudupResourcesAddonsAuthenticationAwsK8s112YamlTemplate,
 	"cloudup/resources/addons/authentication.kope.io/k8s-1.12.yaml":                                       cloudupResourcesAddonsAuthenticationKopeIoK8s112Yaml,
+	"cloudup/resources/addons/aws-cloud-controller.addons.k8s.io/k8s-1.18.yaml.template":                  cloudupResourcesAddonsAwsCloudControllerAddonsK8sIoK8s118YamlTemplate,
 	"cloudup/resources/addons/cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml.template":                    cloudupResourcesAddonsClusterAutoscalerAddonsK8sIoK8s115YamlTemplate,
 	"cloudup/resources/addons/core.addons.k8s.io/addon.yaml":                                              cloudupResourcesAddonsCoreAddonsK8sIoAddonYaml,
 	"cloudup/resources/addons/core.addons.k8s.io/k8s-1.12.yaml.template":                                  cloudupResourcesAddonsCoreAddonsK8sIoK8s112YamlTemplate,
@ -16053,6 +16234,9 @@ var _bintree = &bintree{nil, map[string]*bintree{
 				"authentication.kope.io": {nil, map[string]*bintree{
 					"k8s-1.12.yaml": {cloudupResourcesAddonsAuthenticationKopeIoK8s112Yaml, map[string]*bintree{}},
 				}},
+				"aws-cloud-controller.addons.k8s.io": {nil, map[string]*bintree{
+					"k8s-1.18.yaml.template": {cloudupResourcesAddonsAwsCloudControllerAddonsK8sIoK8s118YamlTemplate, map[string]*bintree{}},
+				}},
 				"cluster-autoscaler.addons.k8s.io": {nil, map[string]*bintree{
 					"k8s-1.15.yaml.template": {cloudupResourcesAddonsClusterAutoscalerAddonsK8sIoK8s115YamlTemplate, map[string]*bintree{}},
 				}},
--- a/upup/models/cloudup/resources/addons/aws-cloud-controller.addons.k8s.io/k8s-1.18.yaml.template
+++ b/upup/models/cloudup/resources/addons/aws-cloud-controller.addons.k8s.io/k8s-1.18.yaml.template
@ -0,0 +1,162 @@
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: aws-cloud-controller-manager
+  namespace: kube-system
+  labels:
+    k8s-app: aws-cloud-controller-manager
+spec:
+  selector:
+    matchLabels:
+      k8s-app: aws-cloud-controller-manager
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        k8s-app: aws-cloud-controller-manager
+    spec:
+      nodeSelector:
+        node-role.kubernetes.io/master: ""
+      tolerations:
+      - key: node.cloudprovider.kubernetes.io/uninitialized
+        value: "true"
+        effect: NoSchedule
+      - key: node-role.kubernetes.io/master
+        effect: NoSchedule
+      serviceAccountName: cloud-controller-manager
+      containers:
+        - name: aws-cloud-controller-manager
+          image: {{ if .ExternalCloudControllerManager.Image }}{{ .ExternalCloudControllerManager.Image }}{{ else }}gcr.io/k8s-staging-provider-aws/cloud-controller-manager:{{AWSCCMTag}}{{ end }}
+          args:
+{{- range $arg := CloudControllerConfigArgv }}
+            - {{ $arg }}
+{{- end }}
+          resources:
+            requests:
+              cpu: 200m
+      hostNetwork: true
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: cloud-controller-manager
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: cloud-controller-manager:apiserver-authentication-reader
+  namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: extension-apiserver-authentication-reader
+subjects:
+- apiGroup: ""
+  kind: ServiceAccount
+  name: cloud-controller-manager
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: system:cloud-controller-manager
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - create
+  - patch
+  - update
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - '*'
+- apiGroups:
+  - ""
+  resources:
+  - nodes/status
+  verbs:
+  - patch
+- apiGroups:
+  - ""
+  resources:
+  - services
+  verbs:
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - services/status
+  verbs:
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - serviceaccounts
+  verbs:
+  - create
+  - get
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumes
+  verbs:
+  - get
+  - list
+  - update
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - endpoints
+  verbs:
+  - create
+  - get
+  - list
+  - watch
+  - update
+- apiGroups:
+  - coordination.k8s.io
+  resources:
+  - leases
+  verbs:
+  - create
+  - get
+  - list
+  - watch
+  - update
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
+  - list
+  - watch
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: system:cloud-controller-manager
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:cloud-controller-manager
+subjects:
+- apiGroup: ""
+  kind: ServiceAccount
+  name: cloud-controller-manager
+  namespace: kube-system
+
--- a/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go
+++ b/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go
@ -67,6 +67,7 @@ func (b *BootstrapChannelBuilder) Build(c *fi.ModelBuilderContext) error {
 		}
 		name := b.Cluster.ObjectMeta.Name + "-addons-" + key
 		manifestPath := "addons/" + *a.Manifest
+		klog.V(4).Infof("Addon %q", name)

 		manifestResource := b.templates.Find(manifestPath)
 		if manifestResource == nil {
@ -983,6 +984,28 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann
 		}
 	}

+	if kops.CloudProviderID(b.Cluster.Spec.CloudProvider) == kops.CloudProviderAWS {
+		key := "aws-cloud-controller.addons.k8s.io"
+
+		if b.Cluster.Spec.ExternalCloudControllerManager != nil {
+			// Version refers to the addon configuration.  The CCM tag is given by
+			// the template function AWSCCMTag()
+			version := "1.18.0-kops.1"
+			{
+				id := "k8s-1.18"
+				location := key + "/" + id + ".yaml"
+				addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{
+					Name:              fi.String(key),
+					Version:           fi.String(version),
+					Manifest:          fi.String(location),
+					Selector:          map[string]string{"k8s-addon": key},
+					KubernetesVersion: ">=1.18.0",
+					Id:                id,
+				})
+			}
+		}
+	}
+
 	if b.Cluster.Spec.KubeScheduler.UsePolicyConfigMap != nil {
 		key := "scheduler.addons.k8s.io"
 		version := "1.7.0"
--- a/upup/pkg/fi/cloudup/bootstrapchannelbuilder_test.go
+++ b/upup/pkg/fi/cloudup/bootstrapchannelbuilder_test.go
@ -65,6 +65,20 @@ func TestBootstrapChannelBuilder_PublicJWKS(t *testing.T) {
 	runChannelBuilderTest(t, "public-jwks", []string{"dns-controller.addons.k8s.io-k8s-1.12", "kops-controller.addons.k8s.io-k8s-1.16", "anonymous-issuer-discovery.addons.k8s.io-k8s-1.16"})
 }

+func TestBootstrapChannelBuilder_AWSCloudController(t *testing.T) {
+	h := testutils.NewIntegrationTestHarness(t)
+	defer h.Close()
+
+	h.SetupMockAWS()
+
+	featureflag.ParseFlags("+EnableExternalCloudController")
+	unsetFeatureFlag := func() {
+		featureflag.ParseFlags("-EnableExternalCloudController")
+	}
+	defer unsetFeatureFlag()
+	runChannelBuilderTest(t, "awscloudcontroller", []string{"aws-cloud-controller.addons.k8s.io-k8s-1.18"})
+}
+
 func runChannelBuilderTest(t *testing.T, key string, addonManifests []string) {
 	basedir := path.Join("tests/bootstrapchannelbuilder/", key)

--- a/upup/pkg/fi/cloudup/template_functions.go
+++ b/upup/pkg/fi/cloudup/template_functions.go
@ -122,6 +122,7 @@ func (tf *TemplateFunctions) AddTo(dest template.FuncMap, secretStore fi.SecretS

 	// will return openstack external ccm image location for current kubernetes version
 	dest["OpenStackCCMTag"] = tf.OpenStackCCMTag
+	dest["AWSCCMTag"] = tf.AWSCCMTag
 	dest["ProxyEnv"] = tf.ProxyEnv

 	dest["KopsSystemEnv"] = tf.KopsSystemEnv
@ -532,6 +533,23 @@ func (tf *TemplateFunctions) OpenStackCCMTag() string {
 	return tag
 }

+// AWSCCMTag returns the correct tag for the cloud controller manager based on
+// the Kubernetes Version
+func (tf *TemplateFunctions) AWSCCMTag() (string, error) {
+	var tag string
+	parsed, err := util.ParseKubernetesVersion(tf.Cluster.Spec.KubernetesVersion)
+
+	// Update when we have stable releases
+	if err != nil {
+		return "", fmt.Errorf("failed to parse Kubernetes version from cluster spec: %q", err)
+	} else if parsed.Minor == 18 {
+		tag = "v1.18.0-alpha.1"
+	} else if parsed.Minor == 19 {
+		tag = "v1.19.0-alpha.1"
+	}
+	return tag, nil
+}
+
 // GetNodeInstanceGroups returns a map containing the defined instance groups of role "Node".
 func (tf *TemplateFunctions) GetNodeInstanceGroups() map[string]kops.InstanceGroupSpec {
 	nodegroups := make(map[string]kops.InstanceGroupSpec)
--- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/awscloudcontroller/aws-cloud-controller.addons.k8s.io-k8s-1.18.yaml
+++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/awscloudcontroller/aws-cloud-controller.addons.k8s.io-k8s-1.18.yaml
@ -0,0 +1,169 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  labels:
+    k8s-app: aws-cloud-controller-manager
+  name: aws-cloud-controller-manager
+  namespace: kube-system
+spec:
+  selector:
+    matchLabels:
+      k8s-app: aws-cloud-controller-manager
+  template:
+    metadata:
+      labels:
+        k8s-app: aws-cloud-controller-manager
+    spec:
+      containers:
+      - args:
+        - --v=2
+        - --cloud-provider=aws
+        - --use-service-account-credentials=true
+        image: gcr.io/k8s-staging-provider-aws/cloud-controller-manager:v1.19.0-alpha.1
+        name: aws-cloud-controller-manager
+        resources:
+          requests:
+            cpu: 200m
+      hostNetwork: true
+      nodeSelector:
+        node-role.kubernetes.io/master: ""
+      serviceAccountName: cloud-controller-manager
+      tolerations:
+      - effect: NoSchedule
+        key: node.cloudprovider.kubernetes.io/uninitialized
+        value: "true"
+      - effect: NoSchedule
+        key: node-role.kubernetes.io/master
+  updateStrategy:
+    type: RollingUpdate
+
+---
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: cloud-controller-manager
+  namespace: kube-system
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: cloud-controller-manager:apiserver-authentication-reader
+  namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: extension-apiserver-authentication-reader
+subjects:
+- apiGroup: ""
+  kind: ServiceAccount
+  name: cloud-controller-manager
+  namespace: kube-system
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: system:cloud-controller-manager
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - create
+  - patch
+  - update
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - '*'
+- apiGroups:
+  - ""
+  resources:
+  - nodes/status
+  verbs:
+  - patch
+- apiGroups:
+  - ""
+  resources:
+  - services
+  verbs:
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - services/status
+  verbs:
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - serviceaccounts
+  verbs:
+  - create
+  - get
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumes
+  verbs:
+  - get
+  - list
+  - update
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - endpoints
+  verbs:
+  - create
+  - get
+  - list
+  - watch
+  - update
+- apiGroups:
+  - coordination.k8s.io
+  resources:
+  - leases
+  verbs:
+  - create
+  - get
+  - list
+  - watch
+  - update
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
+  - list
+  - watch
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: system:cloud-controller-manager
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:cloud-controller-manager
+subjects:
+- apiGroup: ""
+  kind: ServiceAccount
+  name: cloud-controller-manager
+  namespace: kube-system
+
--- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/awscloudcontroller/cluster.yaml
+++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/awscloudcontroller/cluster.yaml
@ -0,0 +1,44 @@
+apiVersion: kops.k8s.io/v1alpha2
+kind: Cluster
+metadata:
+  creationTimestamp: "2016-12-10T22:42:27Z"
+  name: minimal.example.com
+spec:
+  addons:
+    - manifest: s3://somebucket/example.yaml
+  kubernetesApiAccess:
+  - 0.0.0.0/0
+  channel: stable
+  cloudProvider: aws
+  configBase: memfs://clusters.example.com/minimal.example.com
+  etcdClusters:
+  - etcdMembers:
+    - instanceGroup: master-us-test-1a
+      name: master-us-test-1a
+    name: main
+  - etcdMembers:
+    - instanceGroup: master-us-test-1a
+      name: master-us-test-1a
+    name: events
+  cloudControllerManager:
+    cloudProvider: aws
+  iam: {}
+  kubernetesVersion: v1.19.2
+  masterInternalName: api.internal.minimal.example.com
+  masterPublicName: api.minimal.example.com
+  additionalSans:
+  - proxy.api.minimal.example.com
+  networkCIDR: 172.20.0.0/16
+  networking:
+    kubenet: {}
+  nonMasqueradeCIDR: 100.64.0.0/10
+  sshAccess:
+    - 0.0.0.0/0
+  topology:
+    masters: public
+    nodes: public
+  subnets:
+  - cidr: 172.20.32.0/19
+    name: us-test-1a
+    type: Public
+    zone: us-test-1a
--- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/awscloudcontroller/manifest.yaml
+++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/awscloudcontroller/manifest.yaml
@ -0,0 +1,71 @@
+kind: Addons
+metadata:
+  creationTimestamp: null
+  name: bootstrap
+spec:
+  addons:
+  - id: k8s-1.16
+    kubernetesVersion: '>=1.16.0-alpha.0'
+    manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml
+    manifestHash: 68aa7f9ecd2d264b5b1dbc5c72d161420f455427
+    name: kops-controller.addons.k8s.io
+    selector:
+      k8s-addon: kops-controller.addons.k8s.io
+    version: 1.19.0-beta.2
+  - manifest: core.addons.k8s.io/v1.4.0.yaml
+    manifestHash: 3ffe9ac576f9eec72e2bdfbd2ea17d56d9b17b90
+    name: core.addons.k8s.io
+    selector:
+      k8s-addon: core.addons.k8s.io
+    version: 1.4.0
+  - id: k8s-1.12
+    manifest: kube-dns.addons.k8s.io/k8s-1.12.yaml
+    manifestHash: db49c98447b9d59dec4fa413461a6614bc6e43e9
+    name: kube-dns.addons.k8s.io
+    selector:
+      k8s-addon: kube-dns.addons.k8s.io
+    version: 1.15.13-kops.3
+  - id: k8s-1.9
+    manifest: kubelet-api.rbac.addons.k8s.io/k8s-1.9.yaml
+    manifestHash: e1508d77cb4e527d7a2939babe36dc350dd83745
+    name: kubelet-api.rbac.addons.k8s.io
+    selector:
+      k8s-addon: kubelet-api.rbac.addons.k8s.io
+    version: v0.0.1
+  - manifest: limit-range.addons.k8s.io/v1.5.0.yaml
+    manifestHash: 2ea50e23f1a5aa41df3724630ac25173738cc90c
+    name: limit-range.addons.k8s.io
+    selector:
+      k8s-addon: limit-range.addons.k8s.io
+    version: 1.5.0
+  - id: k8s-1.12
+    manifest: dns-controller.addons.k8s.io/k8s-1.12.yaml
+    manifestHash: 8b44f8925352ceb1e59d8edd66bb4e0471a11440
+    name: dns-controller.addons.k8s.io
+    selector:
+      k8s-addon: dns-controller.addons.k8s.io
+    version: 1.19.0-beta.2
+  - id: v1.15.0
+    kubernetesVersion: '>=1.15.0'
+    manifest: storage-aws.addons.k8s.io/v1.15.0.yaml
+    manifestHash: 00cf6e46e25b736b2da93c6025ce482474d83904
+    name: storage-aws.addons.k8s.io
+    selector:
+      k8s-addon: storage-aws.addons.k8s.io
+    version: 1.17.0
+  - id: v1.7.0
+    kubernetesVersion: <1.15.0
+    manifest: storage-aws.addons.k8s.io/v1.7.0.yaml
+    manifestHash: 62705a596142e6cc283280e8aa973e51536994c5
+    name: storage-aws.addons.k8s.io
+    selector:
+      k8s-addon: storage-aws.addons.k8s.io
+    version: 1.17.0
+  - id: k8s-1.18
+    kubernetesVersion: '>=1.18.0'
+    manifest: aws-cloud-controller.addons.k8s.io/k8s-1.18.yaml
+    manifestHash: 3edf9b4350921e900c7ad6793b7a39ad4b49d0be
+    name: aws-cloud-controller.addons.k8s.io
+    selector:
+      k8s-addon: aws-cloud-controller.addons.k8s.io
+    version: 1.18.0-kops.1