Simplify the spec and templates a bit

docs/cluster_spec.md

@@ -553,6 +553,16 @@ spec:
       enabled: true
 ```
 
+If you are using kube-proxy in ipvs mode or Cilium as CNI, you have to set the nodeLocalDNS as ClusterDNS.
+
+```yaml
+spec:
+  kubelet:
+    clusterDNS: 169.254.20.10
+  masterKubelet:
+    clusterDNS: 169.254.20.10
+```
+
 ### kubeControllerManager
 This block contains configurations for the `controller-manager`.
 

k8s/crds/kops.k8s.io_clusters.yaml

@@ -1615,12 +1615,6 @@ spec:
                     description: NodeLocalDNS specifies the configuration for the
                       node-local-dns addon
                     properties:
-                      clusterIP:
-                        description: ClusterIP is the cluster ip
-                        type: string
-                      domain:
-                        description: Domain is the dns domain
-                        type: string
                       enabled:
                         description: Disable indicates we do not wish to run the node-local-dns
                           addon
@@ -1630,9 +1624,6 @@ spec:
                           the 169.254.20.0/16 space or any other IP address that can
                           be guaranteed to not collide with any existing IP.
                         type: string
-                      serverIP:
-                        description: ServerIP is the server ip
-                        type: string
                     type: object
                   provider:
                     description: Provider indicates whether CoreDNS or kube-dns will

pkg/apis/kops/cluster.go

@@ -408,14 +408,8 @@ type KubeDNSConfig struct {
 type NodeLocalDNSConfig struct {
 	// Disable indicates we do not wish to run the node-local-dns addon
 	Enabled bool `json:"enabled,omitempty"`
-	// Domain is the dns domain
-	Domain string `json:"domain,omitempty"`
 	// Local listen IP address. It can be any IP in the 169.254.20.0/16 space or any other IP address that can be guaranteed to not collide with any existing IP.
 	LocalIP string `json:"localIP,omitempty"`
-	// ServerIP is the server ip
-	ServerIP string `json:"serverIP,omitempty"`
-	// ClusterIP is the cluster ip
-	ClusterIP string `json:"clusterIP,omitempty"`
 }
 
 // ExternalDNSConfig are options of the dns-controller

pkg/apis/kops/v1alpha2/cluster.go

@@ -406,14 +406,8 @@ type KubeDNSConfig struct {
 type NodeLocalDNSConfig struct {
 	// Disable indicates we do not wish to run the node-local-dns addon
 	Enabled bool `json:"enabled,omitempty"`
-	// Domain is the dns domain
-	Domain string `json:"domain,omitempty"`
 	// Local listen IP address. It can be any IP in the 169.254.20.0/16 space or any other IP address that can be guaranteed to not collide with any existing IP.
 	LocalIP string `json:"localIP,omitempty"`
-	// ServerIP is the server ip
-	ServerIP string `json:"serverIP,omitempty"`
-	// ClusterIP is the cluster ip
-	ClusterIP string `json:"clusterIP,omitempty"`
 }
 
 // ExternalDNSConfig are options of the dns-controller

pkg/apis/kops/v1alpha2/zz_generated.conversion.go

@@ -4838,10 +4838,7 @@ func Convert_kops_NodeAuthorizerSpec_To_v1alpha2_NodeAuthorizerSpec(in *kops.Nod
 
 func autoConvert_v1alpha2_NodeLocalDNSConfig_To_kops_NodeLocalDNSConfig(in *NodeLocalDNSConfig, out *kops.NodeLocalDNSConfig, s conversion.Scope) error {
 	out.Enabled = in.Enabled
-	out.Domain = in.Domain
 	out.LocalIP = in.LocalIP
-	out.ServerIP = in.ServerIP
-	out.ClusterIP = in.ClusterIP
 	return nil
 }
 
@@ -4852,10 +4849,7 @@ func Convert_v1alpha2_NodeLocalDNSConfig_To_kops_NodeLocalDNSConfig(in *NodeLoca
 
 func autoConvert_kops_NodeLocalDNSConfig_To_v1alpha2_NodeLocalDNSConfig(in *kops.NodeLocalDNSConfig, out *NodeLocalDNSConfig, s conversion.Scope) error {
 	out.Enabled = in.Enabled
-	out.Domain = in.Domain
 	out.LocalIP = in.LocalIP
-	out.ServerIP = in.ServerIP
-	out.ClusterIP = in.ClusterIP
 	return nil
 }
 

pkg/apis/kops/validation/legacy.go

@@ -277,11 +277,11 @@ func ValidateCluster(c *kops.Cluster, strict bool) field.ErrorList {
 					allErrs = append(allErrs, field.Forbidden(fieldSpec.Child("kubeDNS", "serverIP"), fmt.Sprintf("ServiceClusterIPRange %q must contain the DNS Server IP %q", c.Spec.ServiceClusterIPRange, address)))
 				}
 				if !featureflag.ExperimentalClusterDNS.Enabled() {
-					if c.Spec.Kubelet != nil && c.Spec.Kubelet.ClusterDNS != c.Spec.KubeDNS.ServerIP {
+					if isExperimentalClusterDNS(c.Spec.Kubelet, c.Spec.KubeDNS) {
-						allErrs = append(allErrs, field.Forbidden(fieldSpec.Child("kubeDNS", "serverIP"), "Kubelet ClusterDNS did not match cluster kubeDNS.serverIP"))
+						allErrs = append(allErrs, field.Forbidden(fieldSpec.Child("kubelet", "clusterDNS"), "Kubelet ClusterDNS did not match cluster kubeDNS.serverIP or nodeLocalDNS.localIP"))
 					}
-					if c.Spec.MasterKubelet != nil && c.Spec.MasterKubelet.ClusterDNS != c.Spec.KubeDNS.ServerIP {
+					if isExperimentalClusterDNS(c.Spec.MasterKubelet, c.Spec.KubeDNS) {
-						allErrs = append(allErrs, field.Forbidden(fieldSpec.Child("kubeDNS", "serverIP"), "MasterKubelet ClusterDNS did not match cluster kubeDNS.serverIP"))
+						allErrs = append(allErrs, field.Forbidden(fieldSpec.Child("masterKubelet", "clusterDNS"), "MasterKubelet ClusterDNS did not match cluster kubeDNS.serverIP or nodeLocalDNS.localIP"))
 					}
 				}
 			}
@@ -714,3 +714,9 @@ func validateKubelet(k *kops.KubeletConfigSpec, c *kops.Cluster, kubeletPath *fi
 	}
 	return allErrs
 }
+
+func isExperimentalClusterDNS(k *kops.KubeletConfigSpec, dns *kops.KubeDNSConfig) bool {
+
+	return k != nil && k.ClusterDNS != dns.ServerIP && dns.NodeLocalDNS != nil && k.ClusterDNS != dns.NodeLocalDNS.LocalIP
+
+}

pkg/model/components/kubedns.go

@@ -78,25 +78,8 @@ func (b *KubeDnsOptionsBuilder) BuildOptions(o interface{}) error {
 	if NodeLocalDNS == nil {
 		NodeLocalDNS = &kops.NodeLocalDNSConfig{}
 		NodeLocalDNS.Enabled = false
-	} else if NodeLocalDNS.Enabled {
+	} else if NodeLocalDNS.Enabled && NodeLocalDNS.LocalIP == "" {
-		// https://kubernetes.io/docs/tasks/administer-cluster/nodelocaldns/#configuration
+		NodeLocalDNS.LocalIP = "169.254.20.10"
-		NodeLocalDNS.Domain = clusterSpec.ClusterDNSDomain
-
-		switch clusterSpec.KubeProxy.ProxyMode {
-		case "iptables":
-			NodeLocalDNS.ServerIP = clusterSpec.KubeDNS.ServerIP
-			// This will be pushed into the Corefile and replaced by NodeLocal DNSCache at startup
-			NodeLocalDNS.ClusterIP = "__PILLAR__CLUSTER__DNS__"
-
-		case "ipvs":
-			NodeLocalDNS.ServerIP = ""
-			NodeLocalDNS.ClusterIP = clusterSpec.KubeDNS.ServerIP
-
-		default:
-			// the default supposes the kube-proxy working in iptables mode
-			NodeLocalDNS.ServerIP = clusterSpec.KubeDNS.ServerIP
-			NodeLocalDNS.ClusterIP = "__PILLAR__CLUSTER__DNS__"
-		}
 	}
 
 	return nil

upup/models/cloudup/resources/addons/nodelocaldns.addons.k8s.io/k8s-1.12.yaml.template

@@ -42,7 +42,7 @@ metadata:
     addonmanager.kubernetes.io/mode: Reconcile
 data:
   Corefile: |
-    {{ .KubeDNS.NodeLocalDNS.Domain }}:53 {
+    {{ KubeDNS.Domain }}:53 {
         errors
         cache {
           success 9984 30
@@ -50,20 +50,20 @@ data:
         }
         reload
         loop
-        bind {{ or .KubeDNS.NodeLocalDNS.LocalIP "169.254.20.10" }} {{ .KubeDNS.NodeLocalDNS.ServerIP }}
+        bind {{ KubeDNS.NodeLocalDNS.LocalIP }}{{ if NodeLocalDNSServerIP }} {{ NodeLocalDNSServerIP }}{{ end }}
-        forward . {{ .KubeDNS.NodeLocalDNS.ClusterIP }} {
+        forward . {{ NodeLocalDNSClusterIP }} {
           force_tcp
         }
         prometheus :9253
-        health {{ or .KubeDNS.NodeLocalDNS.LocalIP "169.254.20.10" }}:8080
+        health {{ KubeDNS.NodeLocalDNS.LocalIP }}:8080
     }
     in-addr.arpa:53 {
         errors
         cache 30
         reload
         loop
-        bind {{ or .KubeDNS.NodeLocalDNS.LocalIP "169.254.20.10" }} {{ .KubeDNS.NodeLocalDNS.ServerIP }}
+        bind {{ KubeDNS.NodeLocalDNS.LocalIP }}{{ if NodeLocalDNSServerIP }} {{ NodeLocalDNSServerIP }}{{ end }}
-        forward . {{ .KubeDNS.NodeLocalDNS.ClusterIP }} {
+        forward . {{ NodeLocalDNSClusterIP }} {
           force_tcp
         }
         prometheus :9253
@@ -73,8 +73,8 @@ data:
         cache 30
         reload
         loop
-        bind {{ or .KubeDNS.NodeLocalDNS.LocalIP "169.254.20.10" }} {{ .KubeDNS.NodeLocalDNS.ServerIP }}
+        bind {{ KubeDNS.NodeLocalDNS.LocalIP }}{{ if NodeLocalDNSServerIP }} {{ NodeLocalDNSServerIP }}{{ end }}
-        forward . {{ .KubeDNS.NodeLocalDNS.ClusterIP }} {
+        forward . {{ NodeLocalDNSClusterIP }} {
           force_tcp
         }
         prometheus :9253
@@ -84,7 +84,7 @@ data:
         cache 30
         reload
         loop
-        bind {{ or .KubeDNS.NodeLocalDNS.LocalIP "169.254.20.10" }} {{ .KubeDNS.NodeLocalDNS.ServerIP }}
+        bind {{ KubeDNS.NodeLocalDNS.LocalIP }}{{ if NodeLocalDNSServerIP }} {{ NodeLocalDNSServerIP }}{{ end }}
         forward . __PILLAR__UPSTREAM__SERVERS__ {
           force_tcp
         }
@@ -133,10 +133,10 @@ spec:
           requests:
             cpu: 25m
             memory: 5Mi
-        {{ if .KubeDNS.NodeLocalDNS.ServerIP }}
+        {{ if NodeLocalDNSServerIP }}
-        args: [ "-localip", "{{ or .KubeDNS.NodeLocalDNS.LocalIP "169.254.20.10" }},{{ .KubeDNS.NodeLocalDNS.ServerIP }}", "-conf", "/etc/Corefile", "-upstreamsvc", "kube-dns-upstream" ]
+        args: [ "-localip", "{{ .KubeDNS.NodeLocalDNS.LocalIP }},{{ NodeLocalDNSServerIP }}", "-conf", "/etc/Corefile", "-upstreamsvc", "kube-dns-upstream" ]
         {{ else }}
-        args: [ "-localip", "{{ or .KubeDNS.NodeLocalDNS.LocalIP "169.254.20.10" }}", "-conf", "/etc/Corefile", "-upstreamsvc", "kube-dns-upstream" ]
+        args: [ "-localip", "{{ .KubeDNS.NodeLocalDNS.LocalIP }}", "-conf", "/etc/Corefile", "-upstreamsvc", "kube-dns-upstream" ]
         {{ end }}
         securityContext:
           privileged: true
@@ -152,7 +152,7 @@ spec:
           protocol: TCP
         livenessProbe:
           httpGet:
-            host: {{ or .KubeDNS.NodeLocalDNS.LocalIP "169.254.20.10" }}
+            host: {{ .KubeDNS.NodeLocalDNS.LocalIP }}
             path: /health
             port: 8080
           initialDelaySeconds: 60

upup/pkg/fi/cloudup/template_functions.go

@@ -97,6 +97,21 @@ func (tf *TemplateFunctions) AddTo(dest template.FuncMap, secretStore fi.SecretS
 		return tf.cluster.Spec.KubeDNS
 	}
 
+	dest["NodeLocalDNSClusterIP"] = func() string {
+		if tf.cluster.Spec.KubeProxy.ProxyMode == "ipvs" {
+			return tf.cluster.Spec.KubeDNS.ServerIP
+		} else {
+			return "__PILLAR__CLUSTER__DNS__"
+		}
+	}
+	dest["NodeLocalDNSServerIP"] = func() string {
+		if tf.cluster.Spec.KubeProxy.ProxyMode == "ipvs" {
+			return ""
+		} else {
+			return tf.cluster.Spec.KubeDNS.ServerIP
+		}
+	}
+
 	dest["KopsControllerArgv"] = tf.KopsControllerArgv
 	dest["KopsControllerConfig"] = tf.KopsControllerConfig
 	dest["DnsControllerArgv"] = tf.DnsControllerArgv