diff --git a/k8s/crds/kops.k8s.io_clusters.yaml b/k8s/crds/kops.k8s.io_clusters.yaml index ad423bda23..295dfcd8df 100644 --- a/k8s/crds/kops.k8s.io_clusters.yaml +++ b/k8s/crds/kops.k8s.io_clusters.yaml @@ -1231,6 +1231,9 @@ spec: experimentalClusterSigningDuration: description: ExperimentalClusterSigningDuration is the duration that determines the length of duration that the signed certificates will be given. (default 8760h0m0s) type: string + externalCloudVolumePlugin: + description: ExternalCloudVolumePlugin is a fallback mechanism that allows a legacy, in-tree cloudprovider to be used for volume plugins even when an external cloud controller manager is being used. This can be used instead of installing CSI. The value should be the same as is used for the --cloud-provider flag, i.e. "aws". + type: string featureGates: additionalProperties: type: string diff --git a/pkg/apis/kops/componentconfig.go b/pkg/apis/kops/componentconfig.go index 79af4e4454..ba4c655480 100644 --- a/pkg/apis/kops/componentconfig.go +++ b/pkg/apis/kops/componentconfig.go @@ -613,6 +613,10 @@ type KubeControllerManagerConfig struct { AuthorizationKubeconfig string `json:"authorizationKubeconfig,omitempty" flag:"authorization-kubeconfig"` // AuthorizationAlwaysAllowPaths is the list of HTTP paths to skip during authorization AuthorizationAlwaysAllowPaths []string `json:"authorizationAlwaysAllowPaths,omitempty" flag:"authorization-always-allow-paths"` + // ExternalCloudVolumePlugin is a fallback mechanism that allows a legacy, in-tree cloudprovider to be used for volume plugins + // even when an external cloud controller manager is being used. This can be used instead of installing CSI. The value should + // be the same as is used for the --cloud-provider flag, i.e. "aws". + ExternalCloudVolumePlugin string `json:"externalCloudVolumePlugin,omitempty" flag:"external-cloud-volume-plugin"` // EnableProfiling enables profiling via web interface host:port/debug/pprof/ EnableProfiling *bool `json:"enableProfiling,omitempty" flag:"profiling"` diff --git a/pkg/apis/kops/v1alpha2/componentconfig.go b/pkg/apis/kops/v1alpha2/componentconfig.go index b0c0555a0c..858c09eeaa 100644 --- a/pkg/apis/kops/v1alpha2/componentconfig.go +++ b/pkg/apis/kops/v1alpha2/componentconfig.go @@ -614,6 +614,8 @@ type KubeControllerManagerConfig struct { AuthorizationKubeconfig string `json:"authorizationKubeconfig,omitempty" flag:"authorization-kubeconfig"` // AuthorizationAlwaysAllowPaths is the list of HTTP paths to skip during authorization AuthorizationAlwaysAllowPaths []string `json:"authorizationAlwaysAllowPaths,omitempty" flag:"authorization-always-allow-paths"` + // ExternalCloudVolumePlugin is a fallback mechanism that allows a legacy, in-tree cloudprovider to be used for volume plugins even when an external cloud controller manager is being used. This can be used instead of installing CSI. The value should be the same as is used for the --cloud-provider flag, i.e. "aws". + ExternalCloudVolumePlugin string `json:"externalCloudVolumePlugin,omitempty" flag:"external-cloud-volume-plugin"` // EnableProfiling enables profiling via web interface host:port/debug/pprof/ EnableProfiling *bool `json:"enableProfiling,omitempty" flag:"profiling"` diff --git a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go index 473ca0da0d..d915a11dd8 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go @@ -4170,6 +4170,7 @@ func autoConvert_v1alpha2_KubeControllerManagerConfig_To_kops_KubeControllerMana out.AuthenticationKubeconfig = in.AuthenticationKubeconfig out.AuthorizationKubeconfig = in.AuthorizationKubeconfig out.AuthorizationAlwaysAllowPaths = in.AuthorizationAlwaysAllowPaths + out.ExternalCloudVolumePlugin = in.ExternalCloudVolumePlugin out.EnableProfiling = in.EnableProfiling return nil } @@ -4235,6 +4236,7 @@ func autoConvert_kops_KubeControllerManagerConfig_To_v1alpha2_KubeControllerMana out.AuthenticationKubeconfig = in.AuthenticationKubeconfig out.AuthorizationKubeconfig = in.AuthorizationKubeconfig out.AuthorizationAlwaysAllowPaths = in.AuthorizationAlwaysAllowPaths + out.ExternalCloudVolumePlugin = in.ExternalCloudVolumePlugin out.EnableProfiling = in.EnableProfiling return nil } diff --git a/pkg/model/components/kubecontrollermanager.go b/pkg/model/components/kubecontrollermanager.go index 394e5178e5..80e5c91576 100644 --- a/pkg/model/components/kubecontrollermanager.go +++ b/pkg/model/components/kubecontrollermanager.go @@ -98,6 +98,14 @@ func (b *KubeControllerManagerOptionsBuilder) BuildOptions(o interface{}) error if clusterSpec.ExternalCloudControllerManager != nil { kcm.CloudProvider = "external" + + // External cloud provider disables KCM volume controllers, so + // most users would want to either install CSI or pass + // --external-cloud-volume-plugin to the KCM, which runs the + // KCM volume controllers. + if kcm.ExternalCloudVolumePlugin == "" { + klog.Infof("An external cloud controller manager is configured, but ExternalCloudVolumePlugin is not configured for the KCM. This means a CSI plugin must be installed by the user or else volume management might not work.") + } } kcm.LogLevel = 2 diff --git a/upup/models/bindata.go b/upup/models/bindata.go index b1bf14a9af..5eab422566 100644 --- a/upup/models/bindata.go +++ b/upup/models/bindata.go @@ -4,6 +4,7 @@ // upup/models/cloudup/resources/addons/anonymous-issuer-discovery.addons.k8s.io/k8s-1.16.yaml.template // upup/models/cloudup/resources/addons/authentication.aws/k8s-1.12.yaml.template // upup/models/cloudup/resources/addons/authentication.kope.io/k8s-1.12.yaml +// upup/models/cloudup/resources/addons/aws-cloud-controller.addons.k8s.io/k8s-1.18.yaml.template // upup/models/cloudup/resources/addons/cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml.template // upup/models/cloudup/resources/addons/core.addons.k8s.io/addon.yaml // upup/models/cloudup/resources/addons/core.addons.k8s.io/k8s-1.12.yaml.template @@ -573,6 +574,185 @@ func cloudupResourcesAddonsAuthenticationKopeIoK8s112Yaml() (*asset, error) { return a, nil } +var _cloudupResourcesAddonsAwsCloudControllerAddonsK8sIoK8s118YamlTemplate = []byte(`--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: aws-cloud-controller-manager + namespace: kube-system + labels: + k8s-app: aws-cloud-controller-manager +spec: + selector: + matchLabels: + k8s-app: aws-cloud-controller-manager + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + k8s-app: aws-cloud-controller-manager + spec: + nodeSelector: + node-role.kubernetes.io/master: "" + tolerations: + - key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + effect: NoSchedule + - key: node-role.kubernetes.io/master + effect: NoSchedule + serviceAccountName: cloud-controller-manager + containers: + - name: aws-cloud-controller-manager + image: {{ if .ExternalCloudControllerManager.Image }}{{ .ExternalCloudControllerManager.Image }}{{ else }}gcr.io/k8s-staging-provider-aws/cloud-controller-manager:{{AWSCCMTag}}{{ end }} + args: +{{- range $arg := CloudControllerConfigArgv }} + - {{ $arg }} +{{- end }} + resources: + requests: + cpu: 200m + hostNetwork: true +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: cloud-controller-manager + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: cloud-controller-manager:apiserver-authentication-reader + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: +- apiGroup: "" + kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: system:cloud-controller-manager +rules: +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update +- apiGroups: + - "" + resources: + - nodes + verbs: + - '*' +- apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch +- apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch +- apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - get + - list + - watch + - update +- apiGroups: + - "" + resources: + - secrets + verbs: + - list + - watch +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: system:cloud-controller-manager +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager +subjects: +- apiGroup: "" + kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + +`) + +func cloudupResourcesAddonsAwsCloudControllerAddonsK8sIoK8s118YamlTemplateBytes() ([]byte, error) { + return _cloudupResourcesAddonsAwsCloudControllerAddonsK8sIoK8s118YamlTemplate, nil +} + +func cloudupResourcesAddonsAwsCloudControllerAddonsK8sIoK8s118YamlTemplate() (*asset, error) { + bytes, err := cloudupResourcesAddonsAwsCloudControllerAddonsK8sIoK8s118YamlTemplateBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "cloudup/resources/addons/aws-cloud-controller.addons.k8s.io/k8s-1.18.yaml.template", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} + a := &asset{bytes: bytes, info: info} + return a, nil +} + var _cloudupResourcesAddonsClusterAutoscalerAddonsK8sIoK8s115YamlTemplate = []byte(`{{ with .ClusterAutoscaler }} # Sourced from https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler/cloudprovider/aws/examples --- @@ -15954,6 +16134,7 @@ var _bindata = map[string]func() (*asset, error){ "cloudup/resources/addons/anonymous-issuer-discovery.addons.k8s.io/k8s-1.16.yaml.template": cloudupResourcesAddonsAnonymousIssuerDiscoveryAddonsK8sIoK8s116YamlTemplate, "cloudup/resources/addons/authentication.aws/k8s-1.12.yaml.template": cloudupResourcesAddonsAuthenticationAwsK8s112YamlTemplate, "cloudup/resources/addons/authentication.kope.io/k8s-1.12.yaml": cloudupResourcesAddonsAuthenticationKopeIoK8s112Yaml, + "cloudup/resources/addons/aws-cloud-controller.addons.k8s.io/k8s-1.18.yaml.template": cloudupResourcesAddonsAwsCloudControllerAddonsK8sIoK8s118YamlTemplate, "cloudup/resources/addons/cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml.template": cloudupResourcesAddonsClusterAutoscalerAddonsK8sIoK8s115YamlTemplate, "cloudup/resources/addons/core.addons.k8s.io/addon.yaml": cloudupResourcesAddonsCoreAddonsK8sIoAddonYaml, "cloudup/resources/addons/core.addons.k8s.io/k8s-1.12.yaml.template": cloudupResourcesAddonsCoreAddonsK8sIoK8s112YamlTemplate, @@ -16053,6 +16234,9 @@ var _bintree = &bintree{nil, map[string]*bintree{ "authentication.kope.io": {nil, map[string]*bintree{ "k8s-1.12.yaml": {cloudupResourcesAddonsAuthenticationKopeIoK8s112Yaml, map[string]*bintree{}}, }}, + "aws-cloud-controller.addons.k8s.io": {nil, map[string]*bintree{ + "k8s-1.18.yaml.template": {cloudupResourcesAddonsAwsCloudControllerAddonsK8sIoK8s118YamlTemplate, map[string]*bintree{}}, + }}, "cluster-autoscaler.addons.k8s.io": {nil, map[string]*bintree{ "k8s-1.15.yaml.template": {cloudupResourcesAddonsClusterAutoscalerAddonsK8sIoK8s115YamlTemplate, map[string]*bintree{}}, }}, diff --git a/upup/models/cloudup/resources/addons/aws-cloud-controller.addons.k8s.io/k8s-1.18.yaml.template b/upup/models/cloudup/resources/addons/aws-cloud-controller.addons.k8s.io/k8s-1.18.yaml.template new file mode 100644 index 0000000000..2961e14829 --- /dev/null +++ b/upup/models/cloudup/resources/addons/aws-cloud-controller.addons.k8s.io/k8s-1.18.yaml.template @@ -0,0 +1,162 @@ +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: aws-cloud-controller-manager + namespace: kube-system + labels: + k8s-app: aws-cloud-controller-manager +spec: + selector: + matchLabels: + k8s-app: aws-cloud-controller-manager + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + k8s-app: aws-cloud-controller-manager + spec: + nodeSelector: + node-role.kubernetes.io/master: "" + tolerations: + - key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + effect: NoSchedule + - key: node-role.kubernetes.io/master + effect: NoSchedule + serviceAccountName: cloud-controller-manager + containers: + - name: aws-cloud-controller-manager + image: {{ if .ExternalCloudControllerManager.Image }}{{ .ExternalCloudControllerManager.Image }}{{ else }}gcr.io/k8s-staging-provider-aws/cloud-controller-manager:{{AWSCCMTag}}{{ end }} + args: +{{- range $arg := CloudControllerConfigArgv }} + - {{ $arg }} +{{- end }} + resources: + requests: + cpu: 200m + hostNetwork: true +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: cloud-controller-manager + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: cloud-controller-manager:apiserver-authentication-reader + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: +- apiGroup: "" + kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: system:cloud-controller-manager +rules: +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update +- apiGroups: + - "" + resources: + - nodes + verbs: + - '*' +- apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch +- apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch +- apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - get + - list + - watch + - update +- apiGroups: + - "" + resources: + - secrets + verbs: + - list + - watch +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: system:cloud-controller-manager +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager +subjects: +- apiGroup: "" + kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + diff --git a/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go b/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go index 3c82798277..3efd9f129b 100644 --- a/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go +++ b/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go @@ -67,6 +67,7 @@ func (b *BootstrapChannelBuilder) Build(c *fi.ModelBuilderContext) error { } name := b.Cluster.ObjectMeta.Name + "-addons-" + key manifestPath := "addons/" + *a.Manifest + klog.V(4).Infof("Addon %q", name) manifestResource := b.templates.Find(manifestPath) if manifestResource == nil { @@ -983,6 +984,28 @@ func (b *BootstrapChannelBuilder) buildAddons(c *fi.ModelBuilderContext) (*chann } } + if kops.CloudProviderID(b.Cluster.Spec.CloudProvider) == kops.CloudProviderAWS { + key := "aws-cloud-controller.addons.k8s.io" + + if b.Cluster.Spec.ExternalCloudControllerManager != nil { + // Version refers to the addon configuration. The CCM tag is given by + // the template function AWSCCMTag() + version := "1.18.0-kops.1" + { + id := "k8s-1.18" + location := key + "/" + id + ".yaml" + addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{ + Name: fi.String(key), + Version: fi.String(version), + Manifest: fi.String(location), + Selector: map[string]string{"k8s-addon": key}, + KubernetesVersion: ">=1.18.0", + Id: id, + }) + } + } + } + if b.Cluster.Spec.KubeScheduler.UsePolicyConfigMap != nil { key := "scheduler.addons.k8s.io" version := "1.7.0" diff --git a/upup/pkg/fi/cloudup/bootstrapchannelbuilder_test.go b/upup/pkg/fi/cloudup/bootstrapchannelbuilder_test.go index 189e813086..212701f23e 100644 --- a/upup/pkg/fi/cloudup/bootstrapchannelbuilder_test.go +++ b/upup/pkg/fi/cloudup/bootstrapchannelbuilder_test.go @@ -65,6 +65,20 @@ func TestBootstrapChannelBuilder_PublicJWKS(t *testing.T) { runChannelBuilderTest(t, "public-jwks", []string{"dns-controller.addons.k8s.io-k8s-1.12", "kops-controller.addons.k8s.io-k8s-1.16", "anonymous-issuer-discovery.addons.k8s.io-k8s-1.16"}) } +func TestBootstrapChannelBuilder_AWSCloudController(t *testing.T) { + h := testutils.NewIntegrationTestHarness(t) + defer h.Close() + + h.SetupMockAWS() + + featureflag.ParseFlags("+EnableExternalCloudController") + unsetFeatureFlag := func() { + featureflag.ParseFlags("-EnableExternalCloudController") + } + defer unsetFeatureFlag() + runChannelBuilderTest(t, "awscloudcontroller", []string{"aws-cloud-controller.addons.k8s.io-k8s-1.18"}) +} + func runChannelBuilderTest(t *testing.T, key string, addonManifests []string) { basedir := path.Join("tests/bootstrapchannelbuilder/", key) diff --git a/upup/pkg/fi/cloudup/template_functions.go b/upup/pkg/fi/cloudup/template_functions.go index 8bbbd77f43..76b657be61 100644 --- a/upup/pkg/fi/cloudup/template_functions.go +++ b/upup/pkg/fi/cloudup/template_functions.go @@ -122,6 +122,7 @@ func (tf *TemplateFunctions) AddTo(dest template.FuncMap, secretStore fi.SecretS // will return openstack external ccm image location for current kubernetes version dest["OpenStackCCMTag"] = tf.OpenStackCCMTag + dest["AWSCCMTag"] = tf.AWSCCMTag dest["ProxyEnv"] = tf.ProxyEnv dest["KopsSystemEnv"] = tf.KopsSystemEnv @@ -532,6 +533,23 @@ func (tf *TemplateFunctions) OpenStackCCMTag() string { return tag } +// AWSCCMTag returns the correct tag for the cloud controller manager based on +// the Kubernetes Version +func (tf *TemplateFunctions) AWSCCMTag() (string, error) { + var tag string + parsed, err := util.ParseKubernetesVersion(tf.Cluster.Spec.KubernetesVersion) + + // Update when we have stable releases + if err != nil { + return "", fmt.Errorf("failed to parse Kubernetes version from cluster spec: %q", err) + } else if parsed.Minor == 18 { + tag = "v1.18.0-alpha.1" + } else if parsed.Minor == 19 { + tag = "v1.19.0-alpha.1" + } + return tag, nil +} + // GetNodeInstanceGroups returns a map containing the defined instance groups of role "Node". func (tf *TemplateFunctions) GetNodeInstanceGroups() map[string]kops.InstanceGroupSpec { nodegroups := make(map[string]kops.InstanceGroupSpec) diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/awscloudcontroller/aws-cloud-controller.addons.k8s.io-k8s-1.18.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/awscloudcontroller/aws-cloud-controller.addons.k8s.io-k8s-1.18.yaml new file mode 100644 index 0000000000..6d1227c4ba --- /dev/null +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/awscloudcontroller/aws-cloud-controller.addons.k8s.io-k8s-1.18.yaml @@ -0,0 +1,169 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: + k8s-app: aws-cloud-controller-manager + name: aws-cloud-controller-manager + namespace: kube-system +spec: + selector: + matchLabels: + k8s-app: aws-cloud-controller-manager + template: + metadata: + labels: + k8s-app: aws-cloud-controller-manager + spec: + containers: + - args: + - --v=2 + - --cloud-provider=aws + - --use-service-account-credentials=true + image: gcr.io/k8s-staging-provider-aws/cloud-controller-manager:v1.19.0-alpha.1 + name: aws-cloud-controller-manager + resources: + requests: + cpu: 200m + hostNetwork: true + nodeSelector: + node-role.kubernetes.io/master: "" + serviceAccountName: cloud-controller-manager + tolerations: + - effect: NoSchedule + key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + - effect: NoSchedule + key: node-role.kubernetes.io/master + updateStrategy: + type: RollingUpdate + +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: cloud-controller-manager + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: cloud-controller-manager:apiserver-authentication-reader + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: +- apiGroup: "" + kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: system:cloud-controller-manager +rules: +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update +- apiGroups: + - "" + resources: + - nodes + verbs: + - '*' +- apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch +- apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch +- apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - get + - list + - watch + - update +- apiGroups: + - "" + resources: + - secrets + verbs: + - list + - watch + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: system:cloud-controller-manager +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager +subjects: +- apiGroup: "" + kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/awscloudcontroller/cluster.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/awscloudcontroller/cluster.yaml new file mode 100644 index 0000000000..b227540ed3 --- /dev/null +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/awscloudcontroller/cluster.yaml @@ -0,0 +1,44 @@ +apiVersion: kops.k8s.io/v1alpha2 +kind: Cluster +metadata: + creationTimestamp: "2016-12-10T22:42:27Z" + name: minimal.example.com +spec: + addons: + - manifest: s3://somebucket/example.yaml + kubernetesApiAccess: + - 0.0.0.0/0 + channel: stable + cloudProvider: aws + configBase: memfs://clusters.example.com/minimal.example.com + etcdClusters: + - etcdMembers: + - instanceGroup: master-us-test-1a + name: master-us-test-1a + name: main + - etcdMembers: + - instanceGroup: master-us-test-1a + name: master-us-test-1a + name: events + cloudControllerManager: + cloudProvider: aws + iam: {} + kubernetesVersion: v1.19.2 + masterInternalName: api.internal.minimal.example.com + masterPublicName: api.minimal.example.com + additionalSans: + - proxy.api.minimal.example.com + networkCIDR: 172.20.0.0/16 + networking: + kubenet: {} + nonMasqueradeCIDR: 100.64.0.0/10 + sshAccess: + - 0.0.0.0/0 + topology: + masters: public + nodes: public + subnets: + - cidr: 172.20.32.0/19 + name: us-test-1a + type: Public + zone: us-test-1a diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/awscloudcontroller/manifest.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/awscloudcontroller/manifest.yaml new file mode 100644 index 0000000000..7891f8c0e0 --- /dev/null +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/awscloudcontroller/manifest.yaml @@ -0,0 +1,71 @@ +kind: Addons +metadata: + creationTimestamp: null + name: bootstrap +spec: + addons: + - id: k8s-1.16 + kubernetesVersion: '>=1.16.0-alpha.0' + manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml + manifestHash: 68aa7f9ecd2d264b5b1dbc5c72d161420f455427 + name: kops-controller.addons.k8s.io + selector: + k8s-addon: kops-controller.addons.k8s.io + version: 1.19.0-beta.2 + - manifest: core.addons.k8s.io/v1.4.0.yaml + manifestHash: 3ffe9ac576f9eec72e2bdfbd2ea17d56d9b17b90 + name: core.addons.k8s.io + selector: + k8s-addon: core.addons.k8s.io + version: 1.4.0 + - id: k8s-1.12 + manifest: kube-dns.addons.k8s.io/k8s-1.12.yaml + manifestHash: db49c98447b9d59dec4fa413461a6614bc6e43e9 + name: kube-dns.addons.k8s.io + selector: + k8s-addon: kube-dns.addons.k8s.io + version: 1.15.13-kops.3 + - id: k8s-1.9 + manifest: kubelet-api.rbac.addons.k8s.io/k8s-1.9.yaml + manifestHash: e1508d77cb4e527d7a2939babe36dc350dd83745 + name: kubelet-api.rbac.addons.k8s.io + selector: + k8s-addon: kubelet-api.rbac.addons.k8s.io + version: v0.0.1 + - manifest: limit-range.addons.k8s.io/v1.5.0.yaml + manifestHash: 2ea50e23f1a5aa41df3724630ac25173738cc90c + name: limit-range.addons.k8s.io + selector: + k8s-addon: limit-range.addons.k8s.io + version: 1.5.0 + - id: k8s-1.12 + manifest: dns-controller.addons.k8s.io/k8s-1.12.yaml + manifestHash: 8b44f8925352ceb1e59d8edd66bb4e0471a11440 + name: dns-controller.addons.k8s.io + selector: + k8s-addon: dns-controller.addons.k8s.io + version: 1.19.0-beta.2 + - id: v1.15.0 + kubernetesVersion: '>=1.15.0' + manifest: storage-aws.addons.k8s.io/v1.15.0.yaml + manifestHash: 00cf6e46e25b736b2da93c6025ce482474d83904 + name: storage-aws.addons.k8s.io + selector: + k8s-addon: storage-aws.addons.k8s.io + version: 1.17.0 + - id: v1.7.0 + kubernetesVersion: <1.15.0 + manifest: storage-aws.addons.k8s.io/v1.7.0.yaml + manifestHash: 62705a596142e6cc283280e8aa973e51536994c5 + name: storage-aws.addons.k8s.io + selector: + k8s-addon: storage-aws.addons.k8s.io + version: 1.17.0 + - id: k8s-1.18 + kubernetesVersion: '>=1.18.0' + manifest: aws-cloud-controller.addons.k8s.io/k8s-1.18.yaml + manifestHash: 3edf9b4350921e900c7ad6793b7a39ad4b49d0be + name: aws-cloud-controller.addons.k8s.io + selector: + k8s-addon: aws-cloud-controller.addons.k8s.io + version: 1.18.0-kops.1