Customize KubeDNS

This PR adds the ability for users to customize the kubedns upstream nameservers and provider stubdomains, as per [here](https://kubernetes.io/docs/tasks/administer-cluster/dns-custom-nameservers/)

pkg/apis/kops/cluster.go

@@ -286,22 +286,24 @@ type LoadBalancerAccessSpec struct {
 
 // KubeDNSConfig defines the kube dns configuration
 type KubeDNSConfig struct {
-	// Image is the name of the docker image to run
-	// Deprecated as this is now in the addon
-	Image string `json:"image,omitempty"`
-	// Replicas is the number of pod replicas
-	// Deprecated as this is now in the addon, and controlled by autoscaler
-	Replicas int `json:"replicas,omitempty"`
-	// Domain is the dns domain
-	Domain string `json:"domain,omitempty"`
-	// ServerIP is the server ip
-	ServerIP string `json:"serverIP,omitempty"`
 	// CacheMaxSize is the maximum entries to keep in dnsmaq
 	CacheMaxSize int `json:"cacheMaxSize,omitempty"`
 	// CacheMaxConcurrent is the maximum number of concurrent queries for dnsmasq
 	CacheMaxConcurrent int `json:"cacheMaxConcurrent,omitempty"`
+	// Domain is the dns domain
+	Domain string `json:"domain,omitempty"`
+	// Image is the name of the docker image to run - @deprecated as this is now in the addon
+	Image string `json:"image,omitempty"`
+	// Replicas is the number of pod replicas - @deprecated as this is now in the addon and controlled by autoscaler
+	Replicas int `json:"replicas,omitempty"`
 	// Provider indicates whether CoreDNS or kube-dns will be the default service discovery.
 	Provider string `json:"provider,omitempty"`
+	// ServerIP is the server ip
+	ServerIP string `json:"serverIP,omitempty"`
+	// StubDomains redirects a domains to another DNS service
+	StubDomains map[string][]string `json:"stubDomains,omitempty"`
+	// UpstreamNameservers sets the upstream nameservers for queries not on the cluster domain
+	UpstreamNameservers []string `json:"upstreamNameservers,omitempty"`
 }
 
 // ExternalDNSConfig are options of the dns-controller

pkg/apis/kops/v1alpha1/cluster.go

@@ -285,22 +285,24 @@ type LoadBalancerAccessSpec struct {
 
 // KubeDNSConfig defines the kube dns configuration
 type KubeDNSConfig struct {
-	// Image is the name of the docker image to run
-	// Deprecated as this is now in the addon
-	Image string `json:"image,omitempty"`
-	// Replicas is the number of pod replicas
-	// Deprecated as this is now in the addon, and controlled by autoscaler
-	Replicas int `json:"replicas,omitempty"`
-	// Domain is the dns domain
-	Domain string `json:"domain,omitempty"`
-	// ServerIP is the server ip
-	ServerIP string `json:"serverIP,omitempty"`
 	// CacheMaxSize is the maximum entries to keep in dnsmaq
 	CacheMaxSize int `json:"cacheMaxSize,omitempty"`
 	// CacheMaxConcurrent is the maximum number of concurrent queries for dnsmasq
 	CacheMaxConcurrent int `json:"cacheMaxConcurrent,omitempty"`
+	// Domain is the dns domain
+	Domain string `json:"domain,omitempty"`
+	// Image is the name of the docker image to run - @deprecated as this is now in the addon
+	Image string `json:"image,omitempty"`
+	// Replicas is the number of pod replicas - @deprecated as this is now in the addon, and controlled by autoscaler
+	Replicas int `json:"replicas,omitempty"`
 	// Provider indicates whether CoreDNS or kube-dns will be the default service discovery.
 	Provider string `json:"provider,omitempty"`
+	// ServerIP is the server ip
+	ServerIP string `json:"serverIP,omitempty"`
+	// StubDomains redirects a domains to another DNS service
+	StubDomains map[string][]string `json:"stubDomains,omitempty"`
+	// UpstreamNameservers sets the upstream nameservers for queries not on the cluster domain
+	UpstreamNameservers []string `json:"upstreamNameservers,omitempty"`
 }
 
 // ExternalDNSConfig are options of the dns-controller

pkg/apis/kops/v1alpha1/zz_generated.conversion.go

@@ -2093,13 +2093,15 @@ func Convert_kops_KubeControllerManagerConfig_To_v1alpha1_KubeControllerManagerC
 }
 
 func autoConvert_v1alpha1_KubeDNSConfig_To_kops_KubeDNSConfig(in *KubeDNSConfig, out *kops.KubeDNSConfig, s conversion.Scope) error {
-	out.Image = in.Image
-	out.Replicas = in.Replicas
-	out.Domain = in.Domain
-	out.ServerIP = in.ServerIP
 	out.CacheMaxSize = in.CacheMaxSize
 	out.CacheMaxConcurrent = in.CacheMaxConcurrent
+	out.Domain = in.Domain
+	out.Image = in.Image
+	out.Replicas = in.Replicas
 	out.Provider = in.Provider
+	out.ServerIP = in.ServerIP
+	out.StubDomains = in.StubDomains
+	out.UpstreamNameservers = in.UpstreamNameservers
 	return nil
 }
 
@@ -2109,13 +2111,15 @@ func Convert_v1alpha1_KubeDNSConfig_To_kops_KubeDNSConfig(in *KubeDNSConfig, out
 }
 
 func autoConvert_kops_KubeDNSConfig_To_v1alpha1_KubeDNSConfig(in *kops.KubeDNSConfig, out *KubeDNSConfig, s conversion.Scope) error {
-	out.Image = in.Image
-	out.Replicas = in.Replicas
-	out.Domain = in.Domain
-	out.ServerIP = in.ServerIP
 	out.CacheMaxSize = in.CacheMaxSize
 	out.CacheMaxConcurrent = in.CacheMaxConcurrent
+	out.Domain = in.Domain
+	out.Image = in.Image
+	out.Replicas = in.Replicas
 	out.Provider = in.Provider
+	out.ServerIP = in.ServerIP
+	out.StubDomains = in.StubDomains
+	out.UpstreamNameservers = in.UpstreamNameservers
 	return nil
 }
 

pkg/apis/kops/v1alpha1/zz_generated.deepcopy.go

@@ -706,7 +706,7 @@ func (in *ClusterSpec) DeepCopyInto(out *ClusterSpec) {
 			*out = nil
 		} else {
 			*out = new(KubeDNSConfig)
-			**out = **in
+			(*in).DeepCopyInto(*out)
 		}
 	}
 	if in.KubeAPIServer != nil {
@@ -2107,6 +2107,23 @@ func (in *KubeControllerManagerConfig) DeepCopy() *KubeControllerManagerConfig {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *KubeDNSConfig) DeepCopyInto(out *KubeDNSConfig) {
 	*out = *in
+	if in.StubDomains != nil {
+		in, out := &in.StubDomains, &out.StubDomains
+		*out = make(map[string][]string, len(*in))
+		for key, val := range *in {
+			if val == nil {
+				(*out)[key] = nil
+			} else {
+				(*out)[key] = make([]string, len(val))
+				copy((*out)[key], val)
+			}
+		}
+	}
+	if in.UpstreamNameservers != nil {
+		in, out := &in.UpstreamNameservers, &out.UpstreamNameservers
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
 	return
 }
 

pkg/apis/kops/v1alpha2/cluster.go

@@ -284,21 +284,26 @@ type LoadBalancerAccessSpec struct {
 	AdditionalSecurityGroups []string         `json:"additionalSecurityGroups,omitempty"`
 }
 
+// KubeDNSConfig defines the kube dns configuration
 type KubeDNSConfig struct {
-	// Image is the name of the docker image to run
-	// Deprecated as this is now in the addon
-	Image string `json:"image,omitempty"`
-
-	// Deprecated as this is now in the addon, and controlled by autoscaler
-	Replicas int    `json:"replicas,omitempty"`
-	Domain   string `json:"domain,omitempty"`
-	ServerIP string `json:"serverIP,omitempty"`
 	// CacheMaxSize is the maximum entries to keep in dnsmaq
 	CacheMaxSize int `json:"cacheMaxSize,omitempty"`
 	// CacheMaxConcurrent is the maximum number of concurrent queries for dnsmasq
 	CacheMaxConcurrent int `json:"cacheMaxConcurrent,omitempty"`
+	// Domain is the dns domain
+	Domain string `json:"domain,omitempty"`
+	// Image is the name of the docker image to run - @deprecated as this is now in the addon
+	Image string `json:"image,omitempty"`
+	// Replicas is the number of pod replicas - @deprecated as this is now in the addon, and controlled by autoscaler
+	Replicas int `json:"replicas,omitempty"`
 	// Provider indicates whether CoreDNS or kube-dns will be the default service discovery.
 	Provider string `json:"provider,omitempty"`
+	// ServerIP is the server ip
+	ServerIP string `json:"serverIP,omitempty"`
+	// StubDomains redirects a domains to another DNS service
+	StubDomains map[string][]string `json:"stubDomains,omitempty"`
+	// UpstreamNameservers sets the upstream nameservers for queries not on the cluster domain
+	UpstreamNameservers []string `json:"upstreamNameservers,omitempty"`
 }
 
 // ExternalDNSConfig are options of the dns-controller

pkg/apis/kops/v1alpha2/zz_generated.conversion.go

@@ -2357,13 +2357,15 @@ func Convert_kops_KubeControllerManagerConfig_To_v1alpha2_KubeControllerManagerC
 }
 
 func autoConvert_v1alpha2_KubeDNSConfig_To_kops_KubeDNSConfig(in *KubeDNSConfig, out *kops.KubeDNSConfig, s conversion.Scope) error {
-	out.Image = in.Image
-	out.Replicas = in.Replicas
-	out.Domain = in.Domain
-	out.ServerIP = in.ServerIP
 	out.CacheMaxSize = in.CacheMaxSize
 	out.CacheMaxConcurrent = in.CacheMaxConcurrent
+	out.Domain = in.Domain
+	out.Image = in.Image
+	out.Replicas = in.Replicas
 	out.Provider = in.Provider
+	out.ServerIP = in.ServerIP
+	out.StubDomains = in.StubDomains
+	out.UpstreamNameservers = in.UpstreamNameservers
 	return nil
 }
 
@@ -2373,13 +2375,15 @@ func Convert_v1alpha2_KubeDNSConfig_To_kops_KubeDNSConfig(in *KubeDNSConfig, out
 }
 
 func autoConvert_kops_KubeDNSConfig_To_v1alpha2_KubeDNSConfig(in *kops.KubeDNSConfig, out *KubeDNSConfig, s conversion.Scope) error {
-	out.Image = in.Image
-	out.Replicas = in.Replicas
-	out.Domain = in.Domain
-	out.ServerIP = in.ServerIP
 	out.CacheMaxSize = in.CacheMaxSize
 	out.CacheMaxConcurrent = in.CacheMaxConcurrent
+	out.Domain = in.Domain
+	out.Image = in.Image
+	out.Replicas = in.Replicas
 	out.Provider = in.Provider
+	out.ServerIP = in.ServerIP
+	out.StubDomains = in.StubDomains
+	out.UpstreamNameservers = in.UpstreamNameservers
 	return nil
 }
 

pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go

@@ -673,7 +673,7 @@ func (in *ClusterSpec) DeepCopyInto(out *ClusterSpec) {
 			*out = nil
 		} else {
 			*out = new(KubeDNSConfig)
-			**out = **in
+			(*in).DeepCopyInto(*out)
 		}
 	}
 	if in.KubeAPIServer != nil {
@@ -2188,6 +2188,23 @@ func (in *KubeControllerManagerConfig) DeepCopy() *KubeControllerManagerConfig {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *KubeDNSConfig) DeepCopyInto(out *KubeDNSConfig) {
 	*out = *in
+	if in.StubDomains != nil {
+		in, out := &in.StubDomains, &out.StubDomains
+		*out = make(map[string][]string, len(*in))
+		for key, val := range *in {
+			if val == nil {
+				(*out)[key] = nil
+			} else {
+				(*out)[key] = make([]string, len(val))
+				copy((*out)[key], val)
+			}
+		}
+	}
+	if in.UpstreamNameservers != nil {
+		in, out := &in.UpstreamNameservers, &out.UpstreamNameservers
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
 	return
 }
 

pkg/apis/kops/validation/legacy.go

@@ -267,27 +267,44 @@ func ValidateCluster(c *kops.Cluster, strict bool) *field.Error {
 		}
 	}
 
-	// Check KubeDNS.ServerIP
+	// @check the custom kubedns options are valid
 	if c.Spec.KubeDNS != nil {
-		serverIPString := c.Spec.KubeDNS.ServerIP
+		if c.Spec.KubeDNS.ServerIP != "" {
-		if serverIPString == "" {
+			address := c.Spec.KubeDNS.ServerIP
-			return field.Required(fieldSpec.Child("KubeDNS", "ServerIP"), "Cluster did not have KubeDNS.ServerIP set")
+			ip := net.ParseIP(address)
+			if ip == nil {
+				return field.Invalid(fieldSpec.Child("kubeDNS", "serverIP"), address, "Cluster had an invalid kubeDNS.serverIP")
+			}
+			if !serviceClusterIPRange.Contains(ip) {
+				return field.Invalid(fieldSpec.Child("kubeDNS", "serverIP"), address, fmt.Sprintf("ServiceClusterIPRange %q must contain the DNS Server IP %q", c.Spec.ServiceClusterIPRange, address))
+			}
+			if c.Spec.Kubelet != nil && c.Spec.Kubelet.ClusterDNS != c.Spec.KubeDNS.ServerIP {
+				return field.Invalid(fieldSpec.Child("kubeDNS", "serverIP"), address, "Kubelet ClusterDNS did not match cluster kubeDNS.serverIP")
+			}
+			if c.Spec.MasterKubelet != nil && c.Spec.MasterKubelet.ClusterDNS != c.Spec.KubeDNS.ServerIP {
+				return field.Invalid(fieldSpec.Child("kubeDNS", "serverIP"), address, "MasterKubelet ClusterDNS did not match cluster kubeDNS.serverIP")
+			}
 		}
 
-		dnsServiceIP := net.ParseIP(serverIPString)
+		// @check the nameservers are valid
-		if dnsServiceIP == nil {
+		for i, x := range c.Spec.KubeDNS.UpstreamNameservers {
-			return field.Invalid(fieldSpec.Child("KubeDNS", "ServerIP"), serverIPString, "Cluster had an invalid KubeDNS.ServerIP")
+			if ip := net.ParseIP(x); ip == nil {
+				return field.Invalid(fieldSpec.Child("kubeDNS", "upstreamNameservers").Index(i), x, "Invalid nameserver given, should be a valid ip address")
+			}
 		}
 
-		if !serviceClusterIPRange.Contains(dnsServiceIP) {
+		// @check the stubdomain if any
-			return field.Invalid(fieldSpec.Child("KubeDNS", "ServerIP"), serverIPString, fmt.Sprintf("ServiceClusterIPRange %q must contain the DNS Server IP %q", c.Spec.ServiceClusterIPRange, serverIPString))
+		if c.Spec.KubeDNS.StubDomains != nil {
-		}
+			for domain, nameservers := range c.Spec.KubeDNS.StubDomains {
-
+				if len(nameservers) <= 0 {
-		if c.Spec.Kubelet != nil && c.Spec.Kubelet.ClusterDNS != c.Spec.KubeDNS.ServerIP {
+					return field.Invalid(fieldSpec.Child("kubeDNS", "stubDomains").Key(domain), domain, "No nameservers specified for the stub domain")
-			return field.Invalid(fieldSpec.Child("KubeDNS", "ServerIP"), serverIPString, "Kubelet ClusterDNS did not match cluster KubeDNS.ServerIP")
+				}
-		}
+				for i, x := range nameservers {
-		if c.Spec.MasterKubelet != nil && c.Spec.MasterKubelet.ClusterDNS != c.Spec.KubeDNS.ServerIP {
+					if ip := net.ParseIP(x); ip == nil {
-			return field.Invalid(fieldSpec.Child("KubeDNS", "ServerIP"), serverIPString, "MasterKubelet ClusterDNS did not match cluster KubeDNS.ServerIP")
+						return field.Invalid(fieldSpec.Child("kubeDNS", "stubDomains").Key(domain).Index(i), x, "Invalid nameserver given, should be a valid ip address")
+					}
+				}
+			}
 		}
 	}
 

pkg/apis/kops/zz_generated.deepcopy.go

@@ -778,7 +778,7 @@ func (in *ClusterSpec) DeepCopyInto(out *ClusterSpec) {
 			*out = nil
 		} else {
 			*out = new(KubeDNSConfig)
-			**out = **in
+			(*in).DeepCopyInto(*out)
 		}
 	}
 	if in.KubeAPIServer != nil {
@@ -2367,6 +2367,23 @@ func (in *KubeControllerManagerConfig) DeepCopy() *KubeControllerManagerConfig {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *KubeDNSConfig) DeepCopyInto(out *KubeDNSConfig) {
 	*out = *in
+	if in.StubDomains != nil {
+		in, out := &in.StubDomains, &out.StubDomains
+		*out = make(map[string][]string, len(*in))
+		for key, val := range *in {
+			if val == nil {
+				(*out)[key] = nil
+			} else {
+				(*out)[key] = make([]string, len(val))
+				copy((*out)[key], val)
+			}
+		}
+	}
+	if in.UpstreamNameservers != nil {
+		in, out := &in.UpstreamNameservers, &out.UpstreamNameservers
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
 	return
 }
 

pkg/model/components/kubedns.go

@@ -28,6 +28,7 @@ type KubeDnsOptionsBuilder struct {
 
 var _ loader.OptionsBuilder = &KubeDnsOptionsBuilder{}
 
+// BuildOptions fills in the kubedns model
 func (b *KubeDnsOptionsBuilder) BuildOptions(o interface{}) error {
 	clusterSpec := o.(*kops.ClusterSpec)
 
@@ -45,12 +46,17 @@ func (b *KubeDnsOptionsBuilder) BuildOptions(o interface{}) error {
 		clusterSpec.KubeDNS.CacheMaxConcurrent = 150
 	}
 
-	ip, err := WellKnownServiceIP(clusterSpec, 10)
+	if clusterSpec.KubeDNS.ServerIP == "" {
-	if err != nil {
+		ip, err := WellKnownServiceIP(clusterSpec, 10)
-		return err
+		if err != nil {
+			return err
+		}
+		clusterSpec.KubeDNS.ServerIP = ip.String()
+	}
+
+	if clusterSpec.KubeDNS.Domain == "" {
+		clusterSpec.KubeDNS.Domain = clusterSpec.ClusterDNSDomain
 	}
-	clusterSpec.KubeDNS.ServerIP = ip.String()
-	clusterSpec.KubeDNS.Domain = clusterSpec.ClusterDNSDomain
 
 	return nil
 }

upup/models/cloudup/resources/addons/kube-dns.addons.k8s.io/k8s-1.6.yaml.template

@@ -12,6 +12,26 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+{{- if or (.KubeDNS.UpstreamNameservers) (.KubeDNS.StubDomains) }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: kube-dns
+  namespace: kube-system
+data:
+  {{- if .KubeDNS.UpstreamNameservers }}
+  upstreamNameservers: |
+    {{ ToJSON .KubeDNS.UpstreamNameservers }}
+  {{- end }}
+  {{- if .KubeDNS.StubDomains }}
+  stubDomains: |
+    {{ ToJSON .KubeDNS.StubDomains }}
+  {{- end }}
+{{- end }}
+
+---
+
 apiVersion: extensions/v1beta1
 kind: Deployment
 metadata:
@@ -126,9 +146,9 @@ spec:
           initialDelaySeconds: 3
           timeoutSeconds: 5
         args:
-        - --domain={{ KubeDNS.Domain }}.
-        - --dns-port=10053
         - --config-dir=/kube-dns-config
+        - --dns-port=10053
+        - --domain={{ KubeDNS.Domain }}.
         - --v=2
         env:
         - name: PROMETHEUS_PORT

upup/pkg/fi/cloudup/template_functions.go

@@ -28,6 +28,7 @@ When defining a new function:
 package cloudup
 
 import (
+	"encoding/json"
 	"fmt"
 	"os"
 	"strconv"
@@ -59,6 +60,7 @@ func (tf *TemplateFunctions) AddTo(dest template.FuncMap) {
 	dest["EtcdScheme"] = tf.EtcdScheme
 	dest["SharedVPC"] = tf.SharedVPC
 	dest["UseEtcdTLS"] = tf.UseEtcdTLS
+	dest["ToJSON"] = tf.ToJSON
 	// Remember that we may be on a different arch from the target.  Hard-code for now.
 	dest["Arch"] = func() string { return "amd64" }
 	dest["replace"] = func(s, find, replace string) string {
@@ -119,6 +121,16 @@ func (tf *TemplateFunctions) UseEtcdTLS() bool {
 	return false
 }
 
+// ToJSON returns a json representation of the struct or on error an empty string
+func (tf *TemplateFunctions) ToJSON(data interface{}) string {
+	encoded, err := json.Marshal(data)
+	if err != nil {
+		return ""
+	}
+
+	return string(encoded)
+}
+
 // EtcdScheme parses and grabs the protocol to the etcd cluster
 func (tf *TemplateFunctions) EtcdScheme() string {
 	if tf.UseEtcdTLS() {