Add CloudControllerConfigArgv template function, Expose CloudControllerConfigArgv function to templates, and testing it.

Exposing CloudControllerConfigArgv function to templates, and testing it.

go.sum

@@ -654,6 +654,7 @@ k8s.io/kubernetes v1.15.3 h1:PqAKiWeebLJQT2/sqTmeezrd2ApvznBVscDt1rBkcV4=
 k8s.io/kubernetes v1.15.3/go.mod h1:4Ggyo4AFgjbIzULOminzUJAvgbzY3j5ysXlW/a0PdcQ=
 k8s.io/legacy-cloud-providers v0.0.0-20190819145509-592c9a46fd00 h1:WW+7ZNYYPL6Mf1WJx+kjo6wowXt7Ztt8eZT8NkFUsqE=
 k8s.io/legacy-cloud-providers v0.0.0-20190819145509-592c9a46fd00/go.mod h1:CQf+5lDQk/abK+N/fTW2zb/QEQBI3x8YluF4sFSHSf4=
+k8s.io/metrics v0.0.0-20190819143841-305e1cef1ab1 h1:vNxfr8VBLdY4B4AkyKUHhFV4X9Vec3QLrb6xFzZ25h4=
 k8s.io/metrics v0.0.0-20190819143841-305e1cef1ab1/go.mod h1:oSHDT3CDT9BmVggrIzzU1nfZArYBO5saiR4MvpmZ+3w=
 k8s.io/repo-infra v0.0.0-20181204233714-00fe14e3d1a3/go.mod h1:+G1xBfZDfVFsm1Tj/HNCvg4QqWx8rJ2Fxpqr1rqp/gQ=
 k8s.io/sample-apiserver v0.0.0-20190819143045-c84c31c165c4/go.mod h1:HGcsHPVwD0ZBoHXB0xh6zdyQnDcFhglCNgM2HK/WVgg=

upup/models/cloudup/resources/addons/openstack.addons.k8s.io/k8s-1.11.yaml.template

@@ -204,10 +204,10 @@ spec:
         image: "{{- .ExternalCloudControllerManager.Image }}"
         args:
           - /bin/openstack-cloud-controller-manager
-          - --v={{ .ExternalCloudControllerManager.LogLevel }}
+{{- range $arg := CloudControllerConfigArgv }}
+          - {{ $arg }}
+{{- end }}
           - --cloud-config=/etc/kubernetes/cloud.config
-          - --cloud-provider=openstack
-          - --use-service-account-credentials=true
           - --address=127.0.0.1
         volumeMounts:
         - mountPath: /etc/kubernetes/cloud.config

upup/models/cloudup/resources/addons/openstack.addons.k8s.io/k8s-1.13.yaml.template

@@ -184,10 +184,10 @@ spec:
         image: "{{- if .ExternalCloudControllerManager.Image -}} {{ .ExternalCloudControllerManager.Image }} {{- else -}} {{OpenStackCCM}} {{- end -}}"
         args:
           - /bin/openstack-cloud-controller-manager
-          - --v={{ if .ExternalCloudControllerManager.LogLevel -}} {{ .ExternalCloudControllerManager.LogLevel }} {{- else }}2{{ end }}
+{{- range $arg := CloudControllerConfigArgv }}
+          - {{ $arg }}
+{{- end }}
           - --cloud-config=/etc/kubernetes/cloud.config
-          - --cloud-provider=openstack
-          - --use-service-account-credentials=true
           - --address=127.0.0.1
         resources:
           requests:

upup/pkg/fi/cloudup/template_functions.go

@@ -101,7 +101,7 @@ func (tf *TemplateFunctions) AddTo(dest template.FuncMap, secretStore fi.SecretS
 	dest["KopsControllerConfig"] = tf.KopsControllerConfig
 	dest["DnsControllerArgv"] = tf.DnsControllerArgv
 	dest["ExternalDnsArgv"] = tf.ExternalDnsArgv
-
+	dest["CloudControllerConfigArgv"] = tf.CloudControllerConfigArgv
 	// TODO: Only for GCE?
 	dest["EncodeGCELabel"] = gce.EncodeGCELabel
 	dest["Region"] = func() string {
@@ -193,6 +193,52 @@ func (tf *TemplateFunctions) GetInstanceGroup(name string) (*kops.InstanceGroup,
 	return nil, fmt.Errorf("InstanceGroup %q not found", name)
 }
 
+// CloudControllerConfigArgv returns the args to external cloud controller
+func (tf *TemplateFunctions) CloudControllerConfigArgv() ([]string, error) {
+	if tf.cluster.Spec.ExternalCloudControllerManager == nil {
+		return nil, fmt.Errorf("ExternalCloudControllerManager is nil")
+	}
+	var argv []string
+
+	if tf.cluster.Spec.ExternalCloudControllerManager.Master != "" {
+		argv = append(argv, fmt.Sprintf("--master=%s", tf.cluster.Spec.ExternalCloudControllerManager.Master))
+	}
+	if tf.cluster.Spec.ExternalCloudControllerManager.LogLevel != 0 {
+		argv = append(argv, fmt.Sprintf("--v=%d", tf.cluster.Spec.ExternalCloudControllerManager.LogLevel))
+	} else {
+		argv = append(argv, "--v=2")
+	}
+	if tf.cluster.Spec.ExternalCloudControllerManager.CloudProvider !=""{
+		argv = append(argv, fmt.Sprintf("--cloud-provider=%s",  tf.cluster.Spec.ExternalCloudControllerManager.CloudProvider))
+	} else if tf.cluster.Spec.CloudProvider != "" {
+		argv = append(argv, fmt.Sprintf("--cloud-provider=%s", tf.cluster.Spec.CloudProvider))
+	} else {
+		return nil, fmt.Errorf("Cloud Provider is not set")
+	}
+	if tf.cluster.Spec.ExternalCloudControllerManager.ClusterName != "" {
+		argv = append(argv, fmt.Sprintf("--cluster-name=%s", tf.cluster.Spec.ExternalCloudControllerManager.ClusterName))
+	}
+	if tf.cluster.Spec.ExternalCloudControllerManager.ClusterCIDR != "" {
+		argv = append(argv, fmt.Sprintf("--cluster-cidr=%s", tf.cluster.Spec.ExternalCloudControllerManager.ClusterCIDR))
+	}
+	if tf.cluster.Spec.ExternalCloudControllerManager.AllocateNodeCIDRs != nil {
+		argv = append(argv, fmt.Sprintf("--allocate-node-cidrs=%t", *tf.cluster.Spec.ExternalCloudControllerManager.AllocateNodeCIDRs))
+	}
+	if tf.cluster.Spec.ExternalCloudControllerManager.ConfigureCloudRoutes != nil {
+		argv = append(argv, fmt.Sprintf("--configure-cloud-routes=%t", *tf.cluster.Spec.ExternalCloudControllerManager.ConfigureCloudRoutes))
+	}
+	if tf.cluster.Spec.ExternalCloudControllerManager.CIDRAllocatorType != nil && *tf.cluster.Spec.ExternalCloudControllerManager.CIDRAllocatorType != ""{
+		argv = append(argv, fmt.Sprintf("--cidr-allocator-type=%s", *tf.cluster.Spec.ExternalCloudControllerManager.CIDRAllocatorType))
+	}
+	if tf.cluster.Spec.ExternalCloudControllerManager.UseServiceAccountCredentials != nil {
+		argv = append(argv, fmt.Sprintf("--use-service-account-credentials=%t", *tf.cluster.Spec.ExternalCloudControllerManager.UseServiceAccountCredentials ))
+	} else {
+		argv = append(argv, fmt.Sprintf("--use-service-account-credentials=%t", true ))
+	}
+
+	return argv, nil
+}
+
 // DnsControllerArgv returns the args to the DNS controller
 func (tf *TemplateFunctions) DnsControllerArgv() ([]string, error) {
 	var argv []string

upup/pkg/fi/cloudup/template_functions_test.go

@@ -0,0 +1,278 @@
+package cloudup
+
+import (
+	"k8s.io/kops/pkg/featureflag"
+	"path/filepath"
+	"io/ioutil"
+	"bytes"
+	"text/template"
+	"k8s.io/kops/pkg/apis/kops"
+	"reflect"
+	"testing"
+	"fmt"
+	"strings"
+	"k8s.io/kops/upup/pkg/fi"
+	"k8s.io/kubernetes/pkg/controller/nodeipam/ipam"
+)
+
+func Test_TemplateFunctions_CloudControllerConfigArgv(t *testing.T) {
+	tests := []struct {
+		desc         string
+		cluster      *kops.Cluster
+		expectedArgv []string
+		expectedError error
+	}{
+		{
+			desc: "Default Configuration",
+			cluster: &kops.Cluster{Spec: kops.ClusterSpec{
+				CloudProvider: string(kops.CloudProviderOpenstack),
+				ExternalCloudControllerManager: &kops.CloudControllerManagerConfig{},
+
+			}},
+			expectedArgv: []string{
+				"--v=2",
+				"--cloud-provider=openstack",
+				"--use-service-account-credentials=true",
+			},
+		},
+		{
+			desc: "Log Level Configuration",
+			cluster: &kops.Cluster{
+				Spec: kops.ClusterSpec{
+					CloudProvider: string(kops.CloudProviderOpenstack),
+					ExternalCloudControllerManager: &kops.CloudControllerManagerConfig{
+						LogLevel: 3,
+					},
+				},
+			},
+			expectedArgv: []string{
+				"--v=3",
+				"--cloud-provider=openstack",
+				"--use-service-account-credentials=true",
+			},
+		},
+		{
+			desc: "ExternalCloudControllerManager CloudProvider Configuration",
+			cluster: &kops.Cluster{
+				Spec: kops.ClusterSpec{
+					ExternalCloudControllerManager: &kops.CloudControllerManagerConfig{
+						CloudProvider: string(kops.CloudProviderOpenstack),
+						LogLevel: 3,
+					},
+				},
+			},
+			expectedArgv: []string{
+				"--v=3",
+				"--cloud-provider=openstack",
+				"--use-service-account-credentials=true",
+			},
+		},
+		{
+			desc: "No CloudProvider Configuration",
+			cluster: &kops.Cluster{
+				Spec: kops.ClusterSpec{
+					ExternalCloudControllerManager: &kops.CloudControllerManagerConfig{
+						LogLevel: 3,
+					},
+				},
+			},
+			expectedError: fmt.Errorf("Cloud Provider is not set"),
+		},
+		{
+			desc: "k8s cluster name",
+			cluster: &kops.Cluster{Spec: kops.ClusterSpec{
+				CloudProvider: string(kops.CloudProviderOpenstack),
+				ExternalCloudControllerManager: &kops.CloudControllerManagerConfig{
+					ClusterName: "k8s",
+				},
+
+
+			}},
+			expectedArgv: []string{
+				"--v=2",
+				"--cloud-provider=openstack",
+				"--cluster-name=k8s",
+				"--use-service-account-credentials=true",
+			},
+		},
+		{
+			desc: "Default Configuration",
+			cluster: &kops.Cluster{Spec: kops.ClusterSpec{
+				CloudProvider:                  string(kops.CloudProviderOpenstack),
+				ExternalCloudControllerManager: &kops.CloudControllerManagerConfig{
+					Master: "127.0.0.1",
+				},
+			}},
+			expectedArgv: []string{
+				"--master=127.0.0.1",
+				"--v=2",
+				"--cloud-provider=openstack",
+				"--use-service-account-credentials=true",
+			},
+		},
+		{
+			desc: "Cluster-cidr Configuration",
+			cluster: &kops.Cluster{Spec: kops.ClusterSpec{
+				CloudProvider:                  string(kops.CloudProviderOpenstack),
+				ExternalCloudControllerManager: &kops.CloudControllerManagerConfig{
+					ClusterCIDR: "10.0.0.0/24",
+				},
+			}},
+			expectedArgv: []string{
+				"--v=2",
+				"--cloud-provider=openstack",
+				"--cluster-cidr=10.0.0.0/24",
+				"--use-service-account-credentials=true",
+			},
+		},
+		{
+			desc: "AllocateNodeCIDRs Configuration",
+			cluster: &kops.Cluster{Spec: kops.ClusterSpec{
+				CloudProvider:                  string(kops.CloudProviderOpenstack),
+				ExternalCloudControllerManager: &kops.CloudControllerManagerConfig{
+					AllocateNodeCIDRs: fi.Bool(true),
+				},
+			}},
+			expectedArgv: []string{
+				"--v=2",
+				"--cloud-provider=openstack",
+				"--allocate-node-cidrs=true",
+				"--use-service-account-credentials=true",
+
+			},
+		},
+		{
+		 			desc: "ConfigureCloudRoutes Configuration",
+		 			cluster: &kops.Cluster{Spec: kops.ClusterSpec{
+		 				CloudProvider:                  string(kops.CloudProviderOpenstack),
+		 				ExternalCloudControllerManager: &kops.CloudControllerManagerConfig{
+		 					ConfigureCloudRoutes: fi.Bool(true),
+		 				},
+		 			}},
+		 			expectedArgv: []string{
+		 				"--v=2",
+		 				"--cloud-provider=openstack",
+		 				"--configure-cloud-routes=true",
+		 				"--use-service-account-credentials=true",
+		 			},
+		 		},
+		 		{
+		 			desc: "CIDRAllocatorType Configuration",
+		 			cluster: &kops.Cluster{Spec: kops.ClusterSpec{
+		 				CloudProvider:                  string(kops.CloudProviderOpenstack),
+		 				ExternalCloudControllerManager: &kops.CloudControllerManagerConfig{
+		 					CIDRAllocatorType: fi.String(string(ipam.RangeAllocatorType)),
+		 				},
+		 			}},
+		 			expectedArgv: []string{
+		 				"--v=2",
+		 				"--cloud-provider=openstack",
+		 				"--cidr-allocator-type=RangeAllocator",
+		 				"--use-service-account-credentials=true",
+		 			},
+		 		},
+		 		{
+		 			desc: "CIDRAllocatorType Configuration",
+		 			cluster: &kops.Cluster{Spec: kops.ClusterSpec{
+		 				CloudProvider:                  string(kops.CloudProviderOpenstack),
+		 				ExternalCloudControllerManager: &kops.CloudControllerManagerConfig{
+		 					UseServiceAccountCredentials: fi.Bool(false),
+		 				},
+		 			}},
+		 			expectedArgv: []string{
+		 				"--v=2",
+		 				"--cloud-provider=openstack",
+		 				"--use-service-account-credentials=false",
+		 			},
+		 		},
+	}
+	for _, testCase := range tests {
+		t.Run(testCase.desc, func(t *testing.T) {
+			tf := &TemplateFunctions{
+				cluster: testCase.cluster,
+			}
+			actual, error := tf.CloudControllerConfigArgv()
+			if !reflect.DeepEqual(error, testCase.expectedError) {
+				t.Errorf("Error differs: %+v instead of %+v", error, testCase.expectedError)
+			}
+			if !reflect.DeepEqual(actual, testCase.expectedArgv) {
+				t.Errorf("Argv differs: %+v instead of %+v", actual, testCase.expectedArgv)
+			}
+		})
+	}
+}
+
+
+func Test_executeTemplate(t *testing.T ) {
+	tests := []struct{
+			desc string
+			cluster *kops.Cluster
+			templateFilename string
+			expectedManifestPath string
+	}{
+		{
+			desc: "test cloud controller template",
+			cluster:  &kops.Cluster{Spec: kops.ClusterSpec{
+				CloudProvider: string(kops.CloudProviderOpenstack),
+				ExternalCloudControllerManager: &kops.CloudControllerManagerConfig{
+					ClusterName: "k8s",
+					Image: "docker.io/k8scloudprovider/openstack-cloud-controller-manager:1.13",
+				},
+		},
+	},
+	templateFilename: "../../../models/cloudup/resources/addons/openstack.addons.k8s.io/k8s-1.13.yaml.template",
+	expectedManifestPath: "./tests/manifests/k8s-1.13.yaml",
+},
+{
+		desc: "test cloud controller template",
+		cluster:  &kops.Cluster{Spec: kops.ClusterSpec{
+			CloudProvider: string(kops.CloudProviderOpenstack),
+			ExternalCloudControllerManager: &kops.CloudControllerManagerConfig{
+				ClusterName: "k8s",
+				Image: "docker.io/k8scloudprovider/openstack-cloud-controller-manager:1.13",
+				},
+			},
+		},
+		templateFilename: "../../../models/cloudup/resources/addons/openstack.addons.k8s.io/k8s-1.11.yaml.template",
+		expectedManifestPath: "./tests/manifests/k8s-1.11.yaml",
+		},
+}
+
+	for _, testCase := range tests {
+		t.Run(testCase.desc, func(t *testing.T) {
+			featureflag.EnableExternalCloudController = featureflag.New("TotalyNotEnableExternalCloudController", featureflag.Bool(true))
+			templateFileAbsolutePath, filePathError := filepath.Abs(testCase.templateFilename)
+			if filePathError != nil {
+				t.Fatalf("error getting path to template: %v", filePathError)
+			}
+
+				tpl := template.New(filepath.Base(templateFileAbsolutePath))
+
+			funcMap := make(template.FuncMap)
+			templateFunctions := TemplateFunctions{cluster: testCase.cluster}
+			templateFunctions.AddTo(funcMap, nil)
+
+			tpl.Funcs(funcMap)
+
+				tpl.Option("missingkey=zero")
+				_, err := tpl.ParseFiles(templateFileAbsolutePath)
+				if err != nil {
+					t.Fatalf("error parsing template %q: %v",  "template", err)
+				}
+				var buffer bytes.Buffer
+				err = tpl.Execute(&buffer, testCase.cluster.Spec)
+				if err != nil {
+					 t.Fatalf("error executing template %q: %v", "template", err)
+				}
+				actualManifest := buffer.Bytes()
+				expectedFileAbsolutePath, _ := filepath.Abs(testCase.expectedManifestPath)
+				expectedManifest, _ := ioutil.ReadFile( expectedFileAbsolutePath )
+
+				actualString := strings.TrimSpace(string(actualManifest))
+				expectedString := strings.TrimSpace(string(expectedManifest))
+				if !reflect.DeepEqual(actualString,expectedString,) {
+					t.Fatalf("Manifests differs: %+v instead of %+v", actualString, expectedString )
+				}
+			})
+	}
+}

upup/pkg/fi/cloudup/tests/manifests/k8s-1.11.yaml

@@ -0,0 +1,221 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: cloud-controller-manager
+  namespace: kube-system
+  labels:
+    k8s-addon: openstack.addons.k8s.io
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: system:cloud-node-controller
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:cloud-node-controller
+subjects:
+- kind: ServiceAccount
+  name: cloud-node-controller
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: system:pvl-controller
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:pvl-controller
+subjects:
+- kind: ServiceAccount
+  name: pvl-controller
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: system:cloud-controller-manager
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:cloud-controller-manager
+subjects:
+- kind: ServiceAccount
+  name: cloud-controller-manager
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: system:cloud-controller-manager
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - create
+  - patch
+  - update
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - '*'
+- apiGroups:
+  - ""
+  resources:
+  - nodes/status
+  verbs:
+  - patch
+- apiGroups:
+  - ""
+  resources:
+  - services
+  verbs:
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - serviceaccounts
+  verbs:
+  - create
+  - get
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumes
+  verbs:
+  - '*'
+- apiGroups:
+  - ""
+  resources:
+  - endpoints
+  verbs:
+  - create
+  - get
+  - list
+  - watch
+  - update
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
+  - list
+  - get
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: system:cloud-node-controller
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - '*'
+- apiGroups:
+  - ""
+  resources:
+  - nodes/status
+  verbs:
+  - patch
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - create
+  - patch
+  - update
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: system:pvl-controller
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumes
+  verbs:
+  - '*'
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - create
+  - patch
+  - update
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  namespace: kube-system
+  name: openstack-cloud-provider
+  labels:
+    k8s-app: openstack-cloud-provider
+    k8s-addon: openstack.addons.k8s.io
+  annotations:
+    scheduler.alpha.kubernetes.io/critical-pod: ""
+spec:
+  updateStrategy:
+    type: RollingUpdate
+  selector:
+    matchLabels:
+      name: openstack-cloud-provider
+  template:
+    metadata:
+      labels:
+        name: openstack-cloud-provider
+    spec:
+      # run on the host network (don't depend on CNI)
+      hostNetwork: true
+      # run on each master node
+      nodeSelector:
+        node-role.kubernetes.io/master: ""
+      securityContext:
+        runAsUser: 1001
+      serviceAccountName: cloud-controller-manager
+      tolerations:
+      - effect: NoSchedule
+        operator: Exists
+      - key: CriticalAddonsOnly
+        operator: Exists
+      containers:
+      - name: openstack-cloud-controller-manager
+        image: "docker.io/k8scloudprovider/openstack-cloud-controller-manager:1.13"
+        args:
+          - /bin/openstack-cloud-controller-manager
+          - --v=2
+          - --cloud-provider=openstack
+          - --cluster-name=k8s
+          - --use-service-account-credentials=true
+          - --cloud-config=/etc/kubernetes/cloud.config
+          - --address=127.0.0.1
+        volumeMounts:
+        - mountPath: /etc/kubernetes/cloud.config
+          name: cloudconfig
+          readOnly: true
+
+      volumes:
+      - hostPath:
+          path: /etc/kubernetes/cloud.config
+        name: cloudconfig

upup/pkg/fi/cloudup/tests/manifests/k8s-1.13.yaml

@@ -0,0 +1,204 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: cloud-controller-manager
+  namespace: kube-system
+  labels:
+    k8s-app: openstack-cloud-provider
+    k8s-addon: openstack.addons.k8s.io
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: system:cloud-node-controller
+  labels:
+    k8s-app: openstack-cloud-provider
+    k8s-addon: openstack.addons.k8s.io
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:cloud-node-controller
+subjects:
+- kind: ServiceAccount
+  name: cloud-node-controller
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: system:cloud-controller-manager
+  labels:
+    k8s-app: openstack-cloud-provider
+    k8s-addon: openstack.addons.k8s.io
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:cloud-controller-manager
+subjects:
+- kind: ServiceAccount
+  name: cloud-controller-manager
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: system:cloud-controller-manager
+  labels:
+    k8s-app: openstack-cloud-provider
+    k8s-addon: openstack.addons.k8s.io
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - create
+  - patch
+  - update
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - '*'
+- apiGroups:
+  - ""
+  resources:
+  - nodes/status
+  verbs:
+  - patch
+- apiGroups:
+  - ""
+  resources:
+  - services
+  verbs:
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - serviceaccounts
+  verbs:
+  - create
+  - get
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumes
+  verbs:
+  - '*'
+- apiGroups:
+  - ""
+  resources:
+  - endpoints
+  verbs:
+  - create
+  - get
+  - list
+  - watch
+  - update
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
+  - list
+  - get
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: system:cloud-node-controller
+  labels:
+    k8s-app: openstack-cloud-provider
+    k8s-addon: openstack.addons.k8s.io
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - '*'
+- apiGroups:
+  - ""
+  resources:
+  - nodes/status
+  verbs:
+  - patch
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - create
+  - patch
+  - update
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  namespace: kube-system
+  name: openstack-cloud-provider
+  labels:
+    k8s-app: openstack-cloud-provider
+    k8s-addon: openstack.addons.k8s.io
+  annotations:
+    scheduler.alpha.kubernetes.io/critical-pod: ""
+spec:
+  updateStrategy:
+    type: RollingUpdate
+  selector:
+    matchLabels:
+      name: openstack-cloud-provider
+  template:
+    metadata:
+      labels:
+        name: openstack-cloud-provider
+    spec:
+      # run on the host network (don't depend on CNI)
+      hostNetwork: true
+      # run on each master node
+      nodeSelector:
+        node-role.kubernetes.io/master: ""
+      securityContext:
+        runAsUser: 1001
+      serviceAccountName: cloud-controller-manager
+      tolerations:
+      - effect: NoSchedule
+        operator: Exists
+      - key: CriticalAddonsOnly
+        operator: Exists
+      containers:
+      - name: openstack-cloud-controller-manager
+        image: "docker.io/k8scloudprovider/openstack-cloud-controller-manager:1.13"
+        args:
+          - /bin/openstack-cloud-controller-manager
+          - --v=2
+          - --cloud-provider=openstack
+          - --cluster-name=k8s
+          - --use-service-account-credentials=true
+          - --cloud-config=/etc/kubernetes/cloud.config
+          - --address=127.0.0.1
+        resources:
+          requests:
+            cpu: 200m
+        volumeMounts:
+        - mountPath: /etc/kubernetes/cloud.config
+          name: cloudconfig
+          readOnly: true
+
+      volumes:
+      - hostPath:
+          path: /etc/kubernetes/cloud.config
+        name: cloudconfig