From d5019a6c115895affecbc9565d21358466edd3ae Mon Sep 17 00:00:00 2001 From: Ole Markus With Date: Tue, 31 Mar 2020 14:42:40 +0200 Subject: [PATCH] Simplify the spec and templates a bit --- docs/cluster_spec.md | 10 +++++++ k8s/crds/kops.k8s.io_clusters.yaml | 9 ------- pkg/apis/kops/cluster.go | 6 ----- pkg/apis/kops/v1alpha2/cluster.go | 6 ----- .../kops/v1alpha2/zz_generated.conversion.go | 6 ----- pkg/apis/kops/validation/legacy.go | 14 +++++++--- pkg/model/components/kubedns.go | 21 ++------------- .../k8s-1.12.yaml.template | 26 +++++++++---------- upup/pkg/fi/cloudup/template_functions.go | 15 +++++++++++ 9 files changed, 50 insertions(+), 63 deletions(-) diff --git a/docs/cluster_spec.md b/docs/cluster_spec.md index a2563fe355..eb23177e08 100644 --- a/docs/cluster_spec.md +++ b/docs/cluster_spec.md @@ -553,6 +553,16 @@ spec: enabled: true ``` +If you are using kube-proxy in ipvs mode or Cilium as CNI, you have to set the nodeLocalDNS as ClusterDNS. + +```yaml +spec: + kubelet: + clusterDNS: 169.254.20.10 + masterKubelet: + clusterDNS: 169.254.20.10 +``` + ### kubeControllerManager This block contains configurations for the `controller-manager`. diff --git a/k8s/crds/kops.k8s.io_clusters.yaml b/k8s/crds/kops.k8s.io_clusters.yaml index 1a32aa887d..42a48128fd 100644 --- a/k8s/crds/kops.k8s.io_clusters.yaml +++ b/k8s/crds/kops.k8s.io_clusters.yaml @@ -1615,12 +1615,6 @@ spec: description: NodeLocalDNS specifies the configuration for the node-local-dns addon properties: - clusterIP: - description: ClusterIP is the cluster ip - type: string - domain: - description: Domain is the dns domain - type: string enabled: description: Disable indicates we do not wish to run the node-local-dns addon @@ -1630,9 +1624,6 @@ spec: the 169.254.20.0/16 space or any other IP address that can be guaranteed to not collide with any existing IP. type: string - serverIP: - description: ServerIP is the server ip - type: string type: object provider: description: Provider indicates whether CoreDNS or kube-dns will diff --git a/pkg/apis/kops/cluster.go b/pkg/apis/kops/cluster.go index ccbe4868a8..ba3c1c407f 100644 --- a/pkg/apis/kops/cluster.go +++ b/pkg/apis/kops/cluster.go @@ -408,14 +408,8 @@ type KubeDNSConfig struct { type NodeLocalDNSConfig struct { // Disable indicates we do not wish to run the node-local-dns addon Enabled bool `json:"enabled,omitempty"` - // Domain is the dns domain - Domain string `json:"domain,omitempty"` // Local listen IP address. It can be any IP in the 169.254.20.0/16 space or any other IP address that can be guaranteed to not collide with any existing IP. LocalIP string `json:"localIP,omitempty"` - // ServerIP is the server ip - ServerIP string `json:"serverIP,omitempty"` - // ClusterIP is the cluster ip - ClusterIP string `json:"clusterIP,omitempty"` } // ExternalDNSConfig are options of the dns-controller diff --git a/pkg/apis/kops/v1alpha2/cluster.go b/pkg/apis/kops/v1alpha2/cluster.go index 25989ece35..53ec1a1069 100644 --- a/pkg/apis/kops/v1alpha2/cluster.go +++ b/pkg/apis/kops/v1alpha2/cluster.go @@ -406,14 +406,8 @@ type KubeDNSConfig struct { type NodeLocalDNSConfig struct { // Disable indicates we do not wish to run the node-local-dns addon Enabled bool `json:"enabled,omitempty"` - // Domain is the dns domain - Domain string `json:"domain,omitempty"` // Local listen IP address. It can be any IP in the 169.254.20.0/16 space or any other IP address that can be guaranteed to not collide with any existing IP. LocalIP string `json:"localIP,omitempty"` - // ServerIP is the server ip - ServerIP string `json:"serverIP,omitempty"` - // ClusterIP is the cluster ip - ClusterIP string `json:"clusterIP,omitempty"` } // ExternalDNSConfig are options of the dns-controller diff --git a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go index 5f9b04ba3b..d614201645 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go @@ -4838,10 +4838,7 @@ func Convert_kops_NodeAuthorizerSpec_To_v1alpha2_NodeAuthorizerSpec(in *kops.Nod func autoConvert_v1alpha2_NodeLocalDNSConfig_To_kops_NodeLocalDNSConfig(in *NodeLocalDNSConfig, out *kops.NodeLocalDNSConfig, s conversion.Scope) error { out.Enabled = in.Enabled - out.Domain = in.Domain out.LocalIP = in.LocalIP - out.ServerIP = in.ServerIP - out.ClusterIP = in.ClusterIP return nil } @@ -4852,10 +4849,7 @@ func Convert_v1alpha2_NodeLocalDNSConfig_To_kops_NodeLocalDNSConfig(in *NodeLoca func autoConvert_kops_NodeLocalDNSConfig_To_v1alpha2_NodeLocalDNSConfig(in *kops.NodeLocalDNSConfig, out *NodeLocalDNSConfig, s conversion.Scope) error { out.Enabled = in.Enabled - out.Domain = in.Domain out.LocalIP = in.LocalIP - out.ServerIP = in.ServerIP - out.ClusterIP = in.ClusterIP return nil } diff --git a/pkg/apis/kops/validation/legacy.go b/pkg/apis/kops/validation/legacy.go index 4658370699..22c68fbbcf 100644 --- a/pkg/apis/kops/validation/legacy.go +++ b/pkg/apis/kops/validation/legacy.go @@ -277,11 +277,11 @@ func ValidateCluster(c *kops.Cluster, strict bool) field.ErrorList { allErrs = append(allErrs, field.Forbidden(fieldSpec.Child("kubeDNS", "serverIP"), fmt.Sprintf("ServiceClusterIPRange %q must contain the DNS Server IP %q", c.Spec.ServiceClusterIPRange, address))) } if !featureflag.ExperimentalClusterDNS.Enabled() { - if c.Spec.Kubelet != nil && c.Spec.Kubelet.ClusterDNS != c.Spec.KubeDNS.ServerIP { - allErrs = append(allErrs, field.Forbidden(fieldSpec.Child("kubeDNS", "serverIP"), "Kubelet ClusterDNS did not match cluster kubeDNS.serverIP")) + if isExperimentalClusterDNS(c.Spec.Kubelet, c.Spec.KubeDNS) { + allErrs = append(allErrs, field.Forbidden(fieldSpec.Child("kubelet", "clusterDNS"), "Kubelet ClusterDNS did not match cluster kubeDNS.serverIP or nodeLocalDNS.localIP")) } - if c.Spec.MasterKubelet != nil && c.Spec.MasterKubelet.ClusterDNS != c.Spec.KubeDNS.ServerIP { - allErrs = append(allErrs, field.Forbidden(fieldSpec.Child("kubeDNS", "serverIP"), "MasterKubelet ClusterDNS did not match cluster kubeDNS.serverIP")) + if isExperimentalClusterDNS(c.Spec.MasterKubelet, c.Spec.KubeDNS) { + allErrs = append(allErrs, field.Forbidden(fieldSpec.Child("masterKubelet", "clusterDNS"), "MasterKubelet ClusterDNS did not match cluster kubeDNS.serverIP or nodeLocalDNS.localIP")) } } } @@ -714,3 +714,9 @@ func validateKubelet(k *kops.KubeletConfigSpec, c *kops.Cluster, kubeletPath *fi } return allErrs } + +func isExperimentalClusterDNS(k *kops.KubeletConfigSpec, dns *kops.KubeDNSConfig) bool { + + return k != nil && k.ClusterDNS != dns.ServerIP && dns.NodeLocalDNS != nil && k.ClusterDNS != dns.NodeLocalDNS.LocalIP + +} diff --git a/pkg/model/components/kubedns.go b/pkg/model/components/kubedns.go index 0a4077bae2..3e56fdad8a 100644 --- a/pkg/model/components/kubedns.go +++ b/pkg/model/components/kubedns.go @@ -78,25 +78,8 @@ func (b *KubeDnsOptionsBuilder) BuildOptions(o interface{}) error { if NodeLocalDNS == nil { NodeLocalDNS = &kops.NodeLocalDNSConfig{} NodeLocalDNS.Enabled = false - } else if NodeLocalDNS.Enabled { - // https://kubernetes.io/docs/tasks/administer-cluster/nodelocaldns/#configuration - NodeLocalDNS.Domain = clusterSpec.ClusterDNSDomain - - switch clusterSpec.KubeProxy.ProxyMode { - case "iptables": - NodeLocalDNS.ServerIP = clusterSpec.KubeDNS.ServerIP - // This will be pushed into the Corefile and replaced by NodeLocal DNSCache at startup - NodeLocalDNS.ClusterIP = "__PILLAR__CLUSTER__DNS__" - - case "ipvs": - NodeLocalDNS.ServerIP = "" - NodeLocalDNS.ClusterIP = clusterSpec.KubeDNS.ServerIP - - default: - // the default supposes the kube-proxy working in iptables mode - NodeLocalDNS.ServerIP = clusterSpec.KubeDNS.ServerIP - NodeLocalDNS.ClusterIP = "__PILLAR__CLUSTER__DNS__" - } + } else if NodeLocalDNS.Enabled && NodeLocalDNS.LocalIP == "" { + NodeLocalDNS.LocalIP = "169.254.20.10" } return nil diff --git a/upup/models/cloudup/resources/addons/nodelocaldns.addons.k8s.io/k8s-1.12.yaml.template b/upup/models/cloudup/resources/addons/nodelocaldns.addons.k8s.io/k8s-1.12.yaml.template index 4797317d9a..e74e7ce084 100644 --- a/upup/models/cloudup/resources/addons/nodelocaldns.addons.k8s.io/k8s-1.12.yaml.template +++ b/upup/models/cloudup/resources/addons/nodelocaldns.addons.k8s.io/k8s-1.12.yaml.template @@ -42,7 +42,7 @@ metadata: addonmanager.kubernetes.io/mode: Reconcile data: Corefile: | - {{ .KubeDNS.NodeLocalDNS.Domain }}:53 { + {{ KubeDNS.Domain }}:53 { errors cache { success 9984 30 @@ -50,20 +50,20 @@ data: } reload loop - bind {{ or .KubeDNS.NodeLocalDNS.LocalIP "169.254.20.10" }} {{ .KubeDNS.NodeLocalDNS.ServerIP }} - forward . {{ .KubeDNS.NodeLocalDNS.ClusterIP }} { + bind {{ KubeDNS.NodeLocalDNS.LocalIP }}{{ if NodeLocalDNSServerIP }} {{ NodeLocalDNSServerIP }}{{ end }} + forward . {{ NodeLocalDNSClusterIP }} { force_tcp } prometheus :9253 - health {{ or .KubeDNS.NodeLocalDNS.LocalIP "169.254.20.10" }}:8080 + health {{ KubeDNS.NodeLocalDNS.LocalIP }}:8080 } in-addr.arpa:53 { errors cache 30 reload loop - bind {{ or .KubeDNS.NodeLocalDNS.LocalIP "169.254.20.10" }} {{ .KubeDNS.NodeLocalDNS.ServerIP }} - forward . {{ .KubeDNS.NodeLocalDNS.ClusterIP }} { + bind {{ KubeDNS.NodeLocalDNS.LocalIP }}{{ if NodeLocalDNSServerIP }} {{ NodeLocalDNSServerIP }}{{ end }} + forward . {{ NodeLocalDNSClusterIP }} { force_tcp } prometheus :9253 @@ -73,8 +73,8 @@ data: cache 30 reload loop - bind {{ or .KubeDNS.NodeLocalDNS.LocalIP "169.254.20.10" }} {{ .KubeDNS.NodeLocalDNS.ServerIP }} - forward . {{ .KubeDNS.NodeLocalDNS.ClusterIP }} { + bind {{ KubeDNS.NodeLocalDNS.LocalIP }}{{ if NodeLocalDNSServerIP }} {{ NodeLocalDNSServerIP }}{{ end }} + forward . {{ NodeLocalDNSClusterIP }} { force_tcp } prometheus :9253 @@ -84,7 +84,7 @@ data: cache 30 reload loop - bind {{ or .KubeDNS.NodeLocalDNS.LocalIP "169.254.20.10" }} {{ .KubeDNS.NodeLocalDNS.ServerIP }} + bind {{ KubeDNS.NodeLocalDNS.LocalIP }}{{ if NodeLocalDNSServerIP }} {{ NodeLocalDNSServerIP }}{{ end }} forward . __PILLAR__UPSTREAM__SERVERS__ { force_tcp } @@ -133,10 +133,10 @@ spec: requests: cpu: 25m memory: 5Mi - {{ if .KubeDNS.NodeLocalDNS.ServerIP }} - args: [ "-localip", "{{ or .KubeDNS.NodeLocalDNS.LocalIP "169.254.20.10" }},{{ .KubeDNS.NodeLocalDNS.ServerIP }}", "-conf", "/etc/Corefile", "-upstreamsvc", "kube-dns-upstream" ] + {{ if NodeLocalDNSServerIP }} + args: [ "-localip", "{{ .KubeDNS.NodeLocalDNS.LocalIP }},{{ NodeLocalDNSServerIP }}", "-conf", "/etc/Corefile", "-upstreamsvc", "kube-dns-upstream" ] {{ else }} - args: [ "-localip", "{{ or .KubeDNS.NodeLocalDNS.LocalIP "169.254.20.10" }}", "-conf", "/etc/Corefile", "-upstreamsvc", "kube-dns-upstream" ] + args: [ "-localip", "{{ .KubeDNS.NodeLocalDNS.LocalIP }}", "-conf", "/etc/Corefile", "-upstreamsvc", "kube-dns-upstream" ] {{ end }} securityContext: privileged: true @@ -152,7 +152,7 @@ spec: protocol: TCP livenessProbe: httpGet: - host: {{ or .KubeDNS.NodeLocalDNS.LocalIP "169.254.20.10" }} + host: {{ .KubeDNS.NodeLocalDNS.LocalIP }} path: /health port: 8080 initialDelaySeconds: 60 diff --git a/upup/pkg/fi/cloudup/template_functions.go b/upup/pkg/fi/cloudup/template_functions.go index 4e4fec025f..060cacbe21 100644 --- a/upup/pkg/fi/cloudup/template_functions.go +++ b/upup/pkg/fi/cloudup/template_functions.go @@ -97,6 +97,21 @@ func (tf *TemplateFunctions) AddTo(dest template.FuncMap, secretStore fi.SecretS return tf.cluster.Spec.KubeDNS } + dest["NodeLocalDNSClusterIP"] = func() string { + if tf.cluster.Spec.KubeProxy.ProxyMode == "ipvs" { + return tf.cluster.Spec.KubeDNS.ServerIP + } else { + return "__PILLAR__CLUSTER__DNS__" + } + } + dest["NodeLocalDNSServerIP"] = func() string { + if tf.cluster.Spec.KubeProxy.ProxyMode == "ipvs" { + return "" + } else { + return tf.cluster.Spec.KubeDNS.ServerIP + } + } + dest["KopsControllerArgv"] = tf.KopsControllerArgv dest["KopsControllerConfig"] = tf.KopsControllerConfig dest["DnsControllerArgv"] = tf.DnsControllerArgv