From d4fbc4f3c345003d0a49f0818c6d3f36bacca245 Mon Sep 17 00:00:00 2001 From: Jesse Haka Date: Mon, 4 Feb 2019 13:34:21 +0200 Subject: [PATCH] external out-of-tree cloudcontrollermanager to openstack --- docs/tutorial/openstack.md | 29 ++++++ .../k8s-1.11.yaml.template | 95 +++++++++++++++++++ .../pkg/fi/cloudup/bootstrapchannelbuilder.go | 79 +++++++++------ 3 files changed, 174 insertions(+), 29 deletions(-) create mode 100644 upup/models/cloudup/resources/addons/openstack.addons.k8s.io/k8s-1.11.yaml.template diff --git a/docs/tutorial/openstack.md b/docs/tutorial/openstack.md index eb59b6ee6f..4a53131607 100644 --- a/docs/tutorial/openstack.md +++ b/docs/tutorial/openstack.md @@ -64,3 +64,32 @@ kops delete cluster my-cluster.k8s.local --yes * `--os-kubelet-ignore-az=true` Nova and Cinder have different availability zones, more information [Kubernetes docs](https://kubernetes.io/docs/concepts/cluster-administration/cloud-providers/#block-storage) * `--os-octavia=true` If Octavia Loadbalancer api should be used instead of old lbaas v2 api. + +# Using external cloud controller manager +If you want use [External CCM](https://github.com/kubernetes/cloud-provider-openstack) in your installation, this section contains instructions what you should do to get it up and running. + +Enable featureflag: + +``` +export KOPS_FEATURE_FLAGS=AlphaAllowOpenstack,+EnableExternalCloudController +``` + +Create cluster without `--yes` flag (or modify existing cluster): + +``` +kops edit cluster +``` + +Add following to clusterspec: + +``` + cloudControllerManager: + image: jesseh/occm:latest <- you can use this or compile your own + logLevel: 2 +``` + +Finally + +``` +kops update cluster --name --yes +``` diff --git a/upup/models/cloudup/resources/addons/openstack.addons.k8s.io/k8s-1.11.yaml.template b/upup/models/cloudup/resources/addons/openstack.addons.k8s.io/k8s-1.11.yaml.template new file mode 100644 index 0000000000..8d5524df97 --- /dev/null +++ b/upup/models/cloudup/resources/addons/openstack.addons.k8s.io/k8s-1.11.yaml.template @@ -0,0 +1,95 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: cloud-controller-manager + namespace: kube-system + labels: + k8s-addon: openstack.addons.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + k8s-addon: openstack.addons.k8s.io + name: system:openstack +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + k8s-addon: openstack.addons.k8s.io + name: system:openstack +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:openstack +subjects: +- kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + namespace: kube-system + name: openstack-cloud-provider + labels: + k8s-app: openstack-cloud-provider + k8s-addon: openstack.addons.k8s.io +spec: + updateStrategy: + type: RollingUpdate + selector: + matchLabels: + name: openstack-cloud-provider + template: + metadata: + labels: + name: openstack-cloud-provider + spec: + # run on the host network (don't depend on CNI) + hostNetwork: true + # run on each master node + nodeSelector: + node-role.kubernetes.io/master: "" + securityContext: + runAsUser: 1001 + serviceAccountName: cloud-controller-manager + tolerations: + # this is required so CCM can bootstrap itself + - key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + effect: NoSchedule + # this is to have the daemonset runnable on master nodes + # the taint may vary depending on your cluster setup + - key: node-role.kubernetes.io/master + effect: NoSchedule + # this is to restrict CCM to only run on master nodes + # the node selector may vary depending on your cluster setup + - key: "CriticalAddonsOnly" + operator: "Exists" + containers: + - name: openstack-cloud-controller-manager + image: "{{- .ExternalCloudControllerManager.Image }}" + args: + - /bin/openstack-cloud-controller-manager + - --v={{ .ExternalCloudControllerManager.LogLevel }} + - --cloud-config=/etc/kubernetes/cloud.config + - --cloud-provider=openstack + - --master=http://localhost:8080 + volumeMounts: + - mountPath: /etc/kubernetes/cloud.config + name: cloudconfig + readOnly: true + volumes: + - hostPath: + path: /etc/kubernetes/cloud.config + name: cloudconfig diff --git a/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go b/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go index 3812537e2e..3c6a441e77 100644 --- a/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go +++ b/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go @@ -1219,40 +1219,61 @@ func (b *BootstrapChannelBuilder) buildManifest() (*channelsapi.Addons, map[stri } if featureflag.EnableExternalCloudController.Enabled() && b.cluster.Spec.ExternalCloudControllerManager != nil { - { - key := "core.addons.k8s.io" - version := "1.7.0" + // cloudprovider specific out-of-tree controller + if kops.CloudProviderID(b.cluster.Spec.CloudProvider) == kops.CloudProviderOpenstack { + { + key := "openstack.addons.k8s.io" + version := "1.11.0" - location := key + "/k8s-1.7.yaml" - id := "k8s-1.7-ccm" + location := key + "/k8s-1.11.yaml" + id := "k8s-1.11-ccm" - addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{ - Name: fi.String(key), - Version: fi.String(version), - Selector: map[string]string{"k8s-addon": key}, - Manifest: fi.String(location), - KubernetesVersion: ">=1.7.0 <1.12.0", - Id: id, - }) - manifests[key+"-"+id] = "addons/" + location - } + addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{ + Name: fi.String(key), + Version: fi.String(version), + Manifest: fi.String(location), + Selector: map[string]string{"k8s-addon": key}, + KubernetesVersion: ">=1.11.0", + Id: id, + }) + manifests[key+"-"+id] = "addons/" + location + } + } else { + { + key := "core.addons.k8s.io" + version := "1.7.0" - { - key := "core.addons.k8s.io" - version := "1.12.0" + location := key + "/k8s-1.7.yaml" + id := "k8s-1.7-ccm" - location := key + "/k8s-1.12.yaml" - id := "k8s-1.12-ccm" + addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{ + Name: fi.String(key), + Version: fi.String(version), + Selector: map[string]string{"k8s-addon": key}, + Manifest: fi.String(location), + KubernetesVersion: ">=1.7.0 <1.12.0", + Id: id, + }) + manifests[key+"-"+id] = "addons/" + location + } - addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{ - Name: fi.String(key), - Version: fi.String(version), - Selector: map[string]string{"k8s-addon": key}, - Manifest: fi.String(location), - KubernetesVersion: ">=1.12.0", - Id: id, - }) - manifests[key+"-"+id] = "addons/" + location + { + key := "core.addons.k8s.io" + version := "1.12.0" + + location := key + "/k8s-1.12.yaml" + id := "k8s-1.12-ccm" + + addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{ + Name: fi.String(key), + Version: fi.String(version), + Selector: map[string]string{"k8s-addon": key}, + Manifest: fi.String(location), + KubernetesVersion: ">=1.12.0", + Id: id, + }) + manifests[key+"-"+id] = "addons/" + location + } } }