Add CloudControllerConfigArgv template function, Expose CloudControllerConfigArgv function to templates, and testing it.

Exposing CloudControllerConfigArgv function to templates, and testing it.
This commit is contained in:
DavidSie 2019-11-14 16:59:48 +01:00 committed by David Siecinski
parent ba22bc2d6c
commit 013f8799fa
7 changed files with 757 additions and 7 deletions

1
go.sum
View File

@ -654,6 +654,7 @@ k8s.io/kubernetes v1.15.3 h1:PqAKiWeebLJQT2/sqTmeezrd2ApvznBVscDt1rBkcV4=
k8s.io/kubernetes v1.15.3/go.mod h1:4Ggyo4AFgjbIzULOminzUJAvgbzY3j5ysXlW/a0PdcQ=
k8s.io/legacy-cloud-providers v0.0.0-20190819145509-592c9a46fd00 h1:WW+7ZNYYPL6Mf1WJx+kjo6wowXt7Ztt8eZT8NkFUsqE=
k8s.io/legacy-cloud-providers v0.0.0-20190819145509-592c9a46fd00/go.mod h1:CQf+5lDQk/abK+N/fTW2zb/QEQBI3x8YluF4sFSHSf4=
k8s.io/metrics v0.0.0-20190819143841-305e1cef1ab1 h1:vNxfr8VBLdY4B4AkyKUHhFV4X9Vec3QLrb6xFzZ25h4=
k8s.io/metrics v0.0.0-20190819143841-305e1cef1ab1/go.mod h1:oSHDT3CDT9BmVggrIzzU1nfZArYBO5saiR4MvpmZ+3w=
k8s.io/repo-infra v0.0.0-20181204233714-00fe14e3d1a3/go.mod h1:+G1xBfZDfVFsm1Tj/HNCvg4QqWx8rJ2Fxpqr1rqp/gQ=
k8s.io/sample-apiserver v0.0.0-20190819143045-c84c31c165c4/go.mod h1:HGcsHPVwD0ZBoHXB0xh6zdyQnDcFhglCNgM2HK/WVgg=

View File

@ -204,10 +204,10 @@ spec:
image: "{{- .ExternalCloudControllerManager.Image }}"
args:
- /bin/openstack-cloud-controller-manager
- --v={{ .ExternalCloudControllerManager.LogLevel }}
{{- range $arg := CloudControllerConfigArgv }}
- {{ $arg }}
{{- end }}
- --cloud-config=/etc/kubernetes/cloud.config
- --cloud-provider=openstack
- --use-service-account-credentials=true
- --address=127.0.0.1
volumeMounts:
- mountPath: /etc/kubernetes/cloud.config

View File

@ -184,10 +184,10 @@ spec:
image: "{{- if .ExternalCloudControllerManager.Image -}} {{ .ExternalCloudControllerManager.Image }} {{- else -}} {{OpenStackCCM}} {{- end -}}"
args:
- /bin/openstack-cloud-controller-manager
- --v={{ if .ExternalCloudControllerManager.LogLevel -}} {{ .ExternalCloudControllerManager.LogLevel }} {{- else }}2{{ end }}
{{- range $arg := CloudControllerConfigArgv }}
- {{ $arg }}
{{- end }}
- --cloud-config=/etc/kubernetes/cloud.config
- --cloud-provider=openstack
- --use-service-account-credentials=true
- --address=127.0.0.1
resources:
requests:

View File

@ -101,7 +101,7 @@ func (tf *TemplateFunctions) AddTo(dest template.FuncMap, secretStore fi.SecretS
dest["KopsControllerConfig"] = tf.KopsControllerConfig
dest["DnsControllerArgv"] = tf.DnsControllerArgv
dest["ExternalDnsArgv"] = tf.ExternalDnsArgv
dest["CloudControllerConfigArgv"] = tf.CloudControllerConfigArgv
// TODO: Only for GCE?
dest["EncodeGCELabel"] = gce.EncodeGCELabel
dest["Region"] = func() string {
@ -193,6 +193,52 @@ func (tf *TemplateFunctions) GetInstanceGroup(name string) (*kops.InstanceGroup,
return nil, fmt.Errorf("InstanceGroup %q not found", name)
}
// CloudControllerConfigArgv returns the args to external cloud controller
func (tf *TemplateFunctions) CloudControllerConfigArgv() ([]string, error) {
if tf.cluster.Spec.ExternalCloudControllerManager == nil {
return nil, fmt.Errorf("ExternalCloudControllerManager is nil")
}
var argv []string
if tf.cluster.Spec.ExternalCloudControllerManager.Master != "" {
argv = append(argv, fmt.Sprintf("--master=%s", tf.cluster.Spec.ExternalCloudControllerManager.Master))
}
if tf.cluster.Spec.ExternalCloudControllerManager.LogLevel != 0 {
argv = append(argv, fmt.Sprintf("--v=%d", tf.cluster.Spec.ExternalCloudControllerManager.LogLevel))
} else {
argv = append(argv, "--v=2")
}
if tf.cluster.Spec.ExternalCloudControllerManager.CloudProvider !=""{
argv = append(argv, fmt.Sprintf("--cloud-provider=%s", tf.cluster.Spec.ExternalCloudControllerManager.CloudProvider))
} else if tf.cluster.Spec.CloudProvider != "" {
argv = append(argv, fmt.Sprintf("--cloud-provider=%s", tf.cluster.Spec.CloudProvider))
} else {
return nil, fmt.Errorf("Cloud Provider is not set")
}
if tf.cluster.Spec.ExternalCloudControllerManager.ClusterName != "" {
argv = append(argv, fmt.Sprintf("--cluster-name=%s", tf.cluster.Spec.ExternalCloudControllerManager.ClusterName))
}
if tf.cluster.Spec.ExternalCloudControllerManager.ClusterCIDR != "" {
argv = append(argv, fmt.Sprintf("--cluster-cidr=%s", tf.cluster.Spec.ExternalCloudControllerManager.ClusterCIDR))
}
if tf.cluster.Spec.ExternalCloudControllerManager.AllocateNodeCIDRs != nil {
argv = append(argv, fmt.Sprintf("--allocate-node-cidrs=%t", *tf.cluster.Spec.ExternalCloudControllerManager.AllocateNodeCIDRs))
}
if tf.cluster.Spec.ExternalCloudControllerManager.ConfigureCloudRoutes != nil {
argv = append(argv, fmt.Sprintf("--configure-cloud-routes=%t", *tf.cluster.Spec.ExternalCloudControllerManager.ConfigureCloudRoutes))
}
if tf.cluster.Spec.ExternalCloudControllerManager.CIDRAllocatorType != nil && *tf.cluster.Spec.ExternalCloudControllerManager.CIDRAllocatorType != ""{
argv = append(argv, fmt.Sprintf("--cidr-allocator-type=%s", *tf.cluster.Spec.ExternalCloudControllerManager.CIDRAllocatorType))
}
if tf.cluster.Spec.ExternalCloudControllerManager.UseServiceAccountCredentials != nil {
argv = append(argv, fmt.Sprintf("--use-service-account-credentials=%t", *tf.cluster.Spec.ExternalCloudControllerManager.UseServiceAccountCredentials ))
} else {
argv = append(argv, fmt.Sprintf("--use-service-account-credentials=%t", true ))
}
return argv, nil
}
// DnsControllerArgv returns the args to the DNS controller
func (tf *TemplateFunctions) DnsControllerArgv() ([]string, error) {
var argv []string

View File

@ -0,0 +1,278 @@
package cloudup
import (
"k8s.io/kops/pkg/featureflag"
"path/filepath"
"io/ioutil"
"bytes"
"text/template"
"k8s.io/kops/pkg/apis/kops"
"reflect"
"testing"
"fmt"
"strings"
"k8s.io/kops/upup/pkg/fi"
"k8s.io/kubernetes/pkg/controller/nodeipam/ipam"
)
func Test_TemplateFunctions_CloudControllerConfigArgv(t *testing.T) {
tests := []struct {
desc string
cluster *kops.Cluster
expectedArgv []string
expectedError error
}{
{
desc: "Default Configuration",
cluster: &kops.Cluster{Spec: kops.ClusterSpec{
CloudProvider: string(kops.CloudProviderOpenstack),
ExternalCloudControllerManager: &kops.CloudControllerManagerConfig{},
}},
expectedArgv: []string{
"--v=2",
"--cloud-provider=openstack",
"--use-service-account-credentials=true",
},
},
{
desc: "Log Level Configuration",
cluster: &kops.Cluster{
Spec: kops.ClusterSpec{
CloudProvider: string(kops.CloudProviderOpenstack),
ExternalCloudControllerManager: &kops.CloudControllerManagerConfig{
LogLevel: 3,
},
},
},
expectedArgv: []string{
"--v=3",
"--cloud-provider=openstack",
"--use-service-account-credentials=true",
},
},
{
desc: "ExternalCloudControllerManager CloudProvider Configuration",
cluster: &kops.Cluster{
Spec: kops.ClusterSpec{
ExternalCloudControllerManager: &kops.CloudControllerManagerConfig{
CloudProvider: string(kops.CloudProviderOpenstack),
LogLevel: 3,
},
},
},
expectedArgv: []string{
"--v=3",
"--cloud-provider=openstack",
"--use-service-account-credentials=true",
},
},
{
desc: "No CloudProvider Configuration",
cluster: &kops.Cluster{
Spec: kops.ClusterSpec{
ExternalCloudControllerManager: &kops.CloudControllerManagerConfig{
LogLevel: 3,
},
},
},
expectedError: fmt.Errorf("Cloud Provider is not set"),
},
{
desc: "k8s cluster name",
cluster: &kops.Cluster{Spec: kops.ClusterSpec{
CloudProvider: string(kops.CloudProviderOpenstack),
ExternalCloudControllerManager: &kops.CloudControllerManagerConfig{
ClusterName: "k8s",
},
}},
expectedArgv: []string{
"--v=2",
"--cloud-provider=openstack",
"--cluster-name=k8s",
"--use-service-account-credentials=true",
},
},
{
desc: "Default Configuration",
cluster: &kops.Cluster{Spec: kops.ClusterSpec{
CloudProvider: string(kops.CloudProviderOpenstack),
ExternalCloudControllerManager: &kops.CloudControllerManagerConfig{
Master: "127.0.0.1",
},
}},
expectedArgv: []string{
"--master=127.0.0.1",
"--v=2",
"--cloud-provider=openstack",
"--use-service-account-credentials=true",
},
},
{
desc: "Cluster-cidr Configuration",
cluster: &kops.Cluster{Spec: kops.ClusterSpec{
CloudProvider: string(kops.CloudProviderOpenstack),
ExternalCloudControllerManager: &kops.CloudControllerManagerConfig{
ClusterCIDR: "10.0.0.0/24",
},
}},
expectedArgv: []string{
"--v=2",
"--cloud-provider=openstack",
"--cluster-cidr=10.0.0.0/24",
"--use-service-account-credentials=true",
},
},
{
desc: "AllocateNodeCIDRs Configuration",
cluster: &kops.Cluster{Spec: kops.ClusterSpec{
CloudProvider: string(kops.CloudProviderOpenstack),
ExternalCloudControllerManager: &kops.CloudControllerManagerConfig{
AllocateNodeCIDRs: fi.Bool(true),
},
}},
expectedArgv: []string{
"--v=2",
"--cloud-provider=openstack",
"--allocate-node-cidrs=true",
"--use-service-account-credentials=true",
},
},
{
desc: "ConfigureCloudRoutes Configuration",
cluster: &kops.Cluster{Spec: kops.ClusterSpec{
CloudProvider: string(kops.CloudProviderOpenstack),
ExternalCloudControllerManager: &kops.CloudControllerManagerConfig{
ConfigureCloudRoutes: fi.Bool(true),
},
}},
expectedArgv: []string{
"--v=2",
"--cloud-provider=openstack",
"--configure-cloud-routes=true",
"--use-service-account-credentials=true",
},
},
{
desc: "CIDRAllocatorType Configuration",
cluster: &kops.Cluster{Spec: kops.ClusterSpec{
CloudProvider: string(kops.CloudProviderOpenstack),
ExternalCloudControllerManager: &kops.CloudControllerManagerConfig{
CIDRAllocatorType: fi.String(string(ipam.RangeAllocatorType)),
},
}},
expectedArgv: []string{
"--v=2",
"--cloud-provider=openstack",
"--cidr-allocator-type=RangeAllocator",
"--use-service-account-credentials=true",
},
},
{
desc: "CIDRAllocatorType Configuration",
cluster: &kops.Cluster{Spec: kops.ClusterSpec{
CloudProvider: string(kops.CloudProviderOpenstack),
ExternalCloudControllerManager: &kops.CloudControllerManagerConfig{
UseServiceAccountCredentials: fi.Bool(false),
},
}},
expectedArgv: []string{
"--v=2",
"--cloud-provider=openstack",
"--use-service-account-credentials=false",
},
},
}
for _, testCase := range tests {
t.Run(testCase.desc, func(t *testing.T) {
tf := &TemplateFunctions{
cluster: testCase.cluster,
}
actual, error := tf.CloudControllerConfigArgv()
if !reflect.DeepEqual(error, testCase.expectedError) {
t.Errorf("Error differs: %+v instead of %+v", error, testCase.expectedError)
}
if !reflect.DeepEqual(actual, testCase.expectedArgv) {
t.Errorf("Argv differs: %+v instead of %+v", actual, testCase.expectedArgv)
}
})
}
}
func Test_executeTemplate(t *testing.T ) {
tests := []struct{
desc string
cluster *kops.Cluster
templateFilename string
expectedManifestPath string
}{
{
desc: "test cloud controller template",
cluster: &kops.Cluster{Spec: kops.ClusterSpec{
CloudProvider: string(kops.CloudProviderOpenstack),
ExternalCloudControllerManager: &kops.CloudControllerManagerConfig{
ClusterName: "k8s",
Image: "docker.io/k8scloudprovider/openstack-cloud-controller-manager:1.13",
},
},
},
templateFilename: "../../../models/cloudup/resources/addons/openstack.addons.k8s.io/k8s-1.13.yaml.template",
expectedManifestPath: "./tests/manifests/k8s-1.13.yaml",
},
{
desc: "test cloud controller template",
cluster: &kops.Cluster{Spec: kops.ClusterSpec{
CloudProvider: string(kops.CloudProviderOpenstack),
ExternalCloudControllerManager: &kops.CloudControllerManagerConfig{
ClusterName: "k8s",
Image: "docker.io/k8scloudprovider/openstack-cloud-controller-manager:1.13",
},
},
},
templateFilename: "../../../models/cloudup/resources/addons/openstack.addons.k8s.io/k8s-1.11.yaml.template",
expectedManifestPath: "./tests/manifests/k8s-1.11.yaml",
},
}
for _, testCase := range tests {
t.Run(testCase.desc, func(t *testing.T) {
featureflag.EnableExternalCloudController = featureflag.New("TotalyNotEnableExternalCloudController", featureflag.Bool(true))
templateFileAbsolutePath, filePathError := filepath.Abs(testCase.templateFilename)
if filePathError != nil {
t.Fatalf("error getting path to template: %v", filePathError)
}
tpl := template.New(filepath.Base(templateFileAbsolutePath))
funcMap := make(template.FuncMap)
templateFunctions := TemplateFunctions{cluster: testCase.cluster}
templateFunctions.AddTo(funcMap, nil)
tpl.Funcs(funcMap)
tpl.Option("missingkey=zero")
_, err := tpl.ParseFiles(templateFileAbsolutePath)
if err != nil {
t.Fatalf("error parsing template %q: %v", "template", err)
}
var buffer bytes.Buffer
err = tpl.Execute(&buffer, testCase.cluster.Spec)
if err != nil {
t.Fatalf("error executing template %q: %v", "template", err)
}
actualManifest := buffer.Bytes()
expectedFileAbsolutePath, _ := filepath.Abs(testCase.expectedManifestPath)
expectedManifest, _ := ioutil.ReadFile( expectedFileAbsolutePath )
actualString := strings.TrimSpace(string(actualManifest))
expectedString := strings.TrimSpace(string(expectedManifest))
if !reflect.DeepEqual(actualString,expectedString,) {
t.Fatalf("Manifests differs: %+v instead of %+v", actualString, expectedString )
}
})
}
}

View File

@ -0,0 +1,221 @@
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: cloud-controller-manager
namespace: kube-system
labels:
k8s-addon: openstack.addons.k8s.io
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: system:cloud-node-controller
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:cloud-node-controller
subjects:
- kind: ServiceAccount
name: cloud-node-controller
namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: system:pvl-controller
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:pvl-controller
subjects:
- kind: ServiceAccount
name: pvl-controller
namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: system:cloud-controller-manager
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:cloud-controller-manager
subjects:
- kind: ServiceAccount
name: cloud-controller-manager
namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: system:cloud-controller-manager
rules:
- apiGroups:
- ""
resources:
- events
verbs:
- create
- patch
- update
- apiGroups:
- ""
resources:
- nodes
verbs:
- '*'
- apiGroups:
- ""
resources:
- nodes/status
verbs:
- patch
- apiGroups:
- ""
resources:
- services
verbs:
- list
- patch
- update
- watch
- apiGroups:
- ""
resources:
- serviceaccounts
verbs:
- create
- get
- apiGroups:
- ""
resources:
- persistentvolumes
verbs:
- '*'
- apiGroups:
- ""
resources:
- endpoints
verbs:
- create
- get
- list
- watch
- update
- apiGroups:
- ""
resources:
- configmaps
verbs:
- get
- list
- watch
- apiGroups:
- ""
resources:
- secrets
verbs:
- list
- get
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: system:cloud-node-controller
rules:
- apiGroups:
- ""
resources:
- nodes
verbs:
- '*'
- apiGroups:
- ""
resources:
- nodes/status
verbs:
- patch
- apiGroups:
- ""
resources:
- events
verbs:
- create
- patch
- update
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: system:pvl-controller
rules:
- apiGroups:
- ""
resources:
- persistentvolumes
verbs:
- '*'
- apiGroups:
- ""
resources:
- events
verbs:
- create
- patch
- update
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
namespace: kube-system
name: openstack-cloud-provider
labels:
k8s-app: openstack-cloud-provider
k8s-addon: openstack.addons.k8s.io
annotations:
scheduler.alpha.kubernetes.io/critical-pod: ""
spec:
updateStrategy:
type: RollingUpdate
selector:
matchLabels:
name: openstack-cloud-provider
template:
metadata:
labels:
name: openstack-cloud-provider
spec:
# run on the host network (don't depend on CNI)
hostNetwork: true
# run on each master node
nodeSelector:
node-role.kubernetes.io/master: ""
securityContext:
runAsUser: 1001
serviceAccountName: cloud-controller-manager
tolerations:
- effect: NoSchedule
operator: Exists
- key: CriticalAddonsOnly
operator: Exists
containers:
- name: openstack-cloud-controller-manager
image: "docker.io/k8scloudprovider/openstack-cloud-controller-manager:1.13"
args:
- /bin/openstack-cloud-controller-manager
- --v=2
- --cloud-provider=openstack
- --cluster-name=k8s
- --use-service-account-credentials=true
- --cloud-config=/etc/kubernetes/cloud.config
- --address=127.0.0.1
volumeMounts:
- mountPath: /etc/kubernetes/cloud.config
name: cloudconfig
readOnly: true
volumes:
- hostPath:
path: /etc/kubernetes/cloud.config
name: cloudconfig

View File

@ -0,0 +1,204 @@
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: cloud-controller-manager
namespace: kube-system
labels:
k8s-app: openstack-cloud-provider
k8s-addon: openstack.addons.k8s.io
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: system:cloud-node-controller
labels:
k8s-app: openstack-cloud-provider
k8s-addon: openstack.addons.k8s.io
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:cloud-node-controller
subjects:
- kind: ServiceAccount
name: cloud-node-controller
namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: system:cloud-controller-manager
labels:
k8s-app: openstack-cloud-provider
k8s-addon: openstack.addons.k8s.io
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:cloud-controller-manager
subjects:
- kind: ServiceAccount
name: cloud-controller-manager
namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: system:cloud-controller-manager
labels:
k8s-app: openstack-cloud-provider
k8s-addon: openstack.addons.k8s.io
rules:
- apiGroups:
- ""
resources:
- events
verbs:
- create
- patch
- update
- apiGroups:
- ""
resources:
- nodes
verbs:
- '*'
- apiGroups:
- ""
resources:
- nodes/status
verbs:
- patch
- apiGroups:
- ""
resources:
- services
verbs:
- list
- patch
- update
- watch
- apiGroups:
- ""
resources:
- serviceaccounts
verbs:
- create
- get
- apiGroups:
- ""
resources:
- persistentvolumes
verbs:
- '*'
- apiGroups:
- ""
resources:
- endpoints
verbs:
- create
- get
- list
- watch
- update
- apiGroups:
- ""
resources:
- configmaps
verbs:
- get
- list
- watch
- apiGroups:
- ""
resources:
- secrets
verbs:
- list
- get
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: system:cloud-node-controller
labels:
k8s-app: openstack-cloud-provider
k8s-addon: openstack.addons.k8s.io
rules:
- apiGroups:
- ""
resources:
- nodes
verbs:
- '*'
- apiGroups:
- ""
resources:
- nodes/status
verbs:
- patch
- apiGroups:
- ""
resources:
- events
verbs:
- create
- patch
- update
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
namespace: kube-system
name: openstack-cloud-provider
labels:
k8s-app: openstack-cloud-provider
k8s-addon: openstack.addons.k8s.io
annotations:
scheduler.alpha.kubernetes.io/critical-pod: ""
spec:
updateStrategy:
type: RollingUpdate
selector:
matchLabels:
name: openstack-cloud-provider
template:
metadata:
labels:
name: openstack-cloud-provider
spec:
# run on the host network (don't depend on CNI)
hostNetwork: true
# run on each master node
nodeSelector:
node-role.kubernetes.io/master: ""
securityContext:
runAsUser: 1001
serviceAccountName: cloud-controller-manager
tolerations:
- effect: NoSchedule
operator: Exists
- key: CriticalAddonsOnly
operator: Exists
containers:
- name: openstack-cloud-controller-manager
image: "docker.io/k8scloudprovider/openstack-cloud-controller-manager:1.13"
args:
- /bin/openstack-cloud-controller-manager
- --v=2
- --cloud-provider=openstack
- --cluster-name=k8s
- --use-service-account-credentials=true
- --cloud-config=/etc/kubernetes/cloud.config
- --address=127.0.0.1
resources:
requests:
cpu: 200m
volumeMounts:
- mountPath: /etc/kubernetes/cloud.config
name: cloudconfig
readOnly: true
volumes:
- hostPath:
path: /etc/kubernetes/cloud.config
name: cloudconfig