From de977e627e6f69a6bc400fc1dbbe376c41212b51 Mon Sep 17 00:00:00 2001 From: Rohith Date: Mon, 19 Mar 2018 20:15:32 +0000 Subject: [PATCH] Customize KubeDNS This PR adds the ability for users to customize the kubedns upstream nameservers and provider stubdomains, as per [here](https://kubernetes.io/docs/tasks/administer-cluster/dns-custom-nameservers/) --- pkg/apis/kops/cluster.go | 22 +++++---- pkg/apis/kops/v1alpha1/cluster.go | 22 +++++---- .../kops/v1alpha1/zz_generated.conversion.go | 20 +++++--- .../kops/v1alpha1/zz_generated.deepcopy.go | 19 ++++++- pkg/apis/kops/v1alpha2/cluster.go | 21 +++++--- .../kops/v1alpha2/zz_generated.conversion.go | 20 +++++--- .../kops/v1alpha2/zz_generated.deepcopy.go | 19 ++++++- pkg/apis/kops/validation/legacy.go | 49 +++++++++++++------ pkg/apis/kops/zz_generated.deepcopy.go | 19 ++++++- pkg/model/components/kubedns.go | 16 ++++-- .../k8s-1.6.yaml.template | 24 ++++++++- upup/pkg/fi/cloudup/template_functions.go | 12 +++++ 12 files changed, 193 insertions(+), 70 deletions(-) diff --git a/pkg/apis/kops/cluster.go b/pkg/apis/kops/cluster.go index 9d223a77f5..de7a34b2b1 100644 --- a/pkg/apis/kops/cluster.go +++ b/pkg/apis/kops/cluster.go @@ -286,22 +286,24 @@ type LoadBalancerAccessSpec struct { // KubeDNSConfig defines the kube dns configuration type KubeDNSConfig struct { - // Image is the name of the docker image to run - // Deprecated as this is now in the addon - Image string `json:"image,omitempty"` - // Replicas is the number of pod replicas - // Deprecated as this is now in the addon, and controlled by autoscaler - Replicas int `json:"replicas,omitempty"` - // Domain is the dns domain - Domain string `json:"domain,omitempty"` - // ServerIP is the server ip - ServerIP string `json:"serverIP,omitempty"` // CacheMaxSize is the maximum entries to keep in dnsmaq CacheMaxSize int `json:"cacheMaxSize,omitempty"` // CacheMaxConcurrent is the maximum number of concurrent queries for dnsmasq CacheMaxConcurrent int `json:"cacheMaxConcurrent,omitempty"` + // Domain is the dns domain + Domain string `json:"domain,omitempty"` + // Image is the name of the docker image to run - @deprecated as this is now in the addon + Image string `json:"image,omitempty"` + // Replicas is the number of pod replicas - @deprecated as this is now in the addon and controlled by autoscaler + Replicas int `json:"replicas,omitempty"` // Provider indicates whether CoreDNS or kube-dns will be the default service discovery. Provider string `json:"provider,omitempty"` + // ServerIP is the server ip + ServerIP string `json:"serverIP,omitempty"` + // StubDomains redirects a domains to another DNS service + StubDomains map[string][]string `json:"stubDomains,omitempty"` + // UpstreamNameservers sets the upstream nameservers for queries not on the cluster domain + UpstreamNameservers []string `json:"upstreamNameservers,omitempty"` } // ExternalDNSConfig are options of the dns-controller diff --git a/pkg/apis/kops/v1alpha1/cluster.go b/pkg/apis/kops/v1alpha1/cluster.go index 376efb1113..ccde03e7cb 100644 --- a/pkg/apis/kops/v1alpha1/cluster.go +++ b/pkg/apis/kops/v1alpha1/cluster.go @@ -285,22 +285,24 @@ type LoadBalancerAccessSpec struct { // KubeDNSConfig defines the kube dns configuration type KubeDNSConfig struct { - // Image is the name of the docker image to run - // Deprecated as this is now in the addon - Image string `json:"image,omitempty"` - // Replicas is the number of pod replicas - // Deprecated as this is now in the addon, and controlled by autoscaler - Replicas int `json:"replicas,omitempty"` - // Domain is the dns domain - Domain string `json:"domain,omitempty"` - // ServerIP is the server ip - ServerIP string `json:"serverIP,omitempty"` // CacheMaxSize is the maximum entries to keep in dnsmaq CacheMaxSize int `json:"cacheMaxSize,omitempty"` // CacheMaxConcurrent is the maximum number of concurrent queries for dnsmasq CacheMaxConcurrent int `json:"cacheMaxConcurrent,omitempty"` + // Domain is the dns domain + Domain string `json:"domain,omitempty"` + // Image is the name of the docker image to run - @deprecated as this is now in the addon + Image string `json:"image,omitempty"` + // Replicas is the number of pod replicas - @deprecated as this is now in the addon, and controlled by autoscaler + Replicas int `json:"replicas,omitempty"` // Provider indicates whether CoreDNS or kube-dns will be the default service discovery. Provider string `json:"provider,omitempty"` + // ServerIP is the server ip + ServerIP string `json:"serverIP,omitempty"` + // StubDomains redirects a domains to another DNS service + StubDomains map[string][]string `json:"stubDomains,omitempty"` + // UpstreamNameservers sets the upstream nameservers for queries not on the cluster domain + UpstreamNameservers []string `json:"upstreamNameservers,omitempty"` } // ExternalDNSConfig are options of the dns-controller diff --git a/pkg/apis/kops/v1alpha1/zz_generated.conversion.go b/pkg/apis/kops/v1alpha1/zz_generated.conversion.go index 732e2b5a1c..b54a244469 100644 --- a/pkg/apis/kops/v1alpha1/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha1/zz_generated.conversion.go @@ -2093,13 +2093,15 @@ func Convert_kops_KubeControllerManagerConfig_To_v1alpha1_KubeControllerManagerC } func autoConvert_v1alpha1_KubeDNSConfig_To_kops_KubeDNSConfig(in *KubeDNSConfig, out *kops.KubeDNSConfig, s conversion.Scope) error { - out.Image = in.Image - out.Replicas = in.Replicas - out.Domain = in.Domain - out.ServerIP = in.ServerIP out.CacheMaxSize = in.CacheMaxSize out.CacheMaxConcurrent = in.CacheMaxConcurrent + out.Domain = in.Domain + out.Image = in.Image + out.Replicas = in.Replicas out.Provider = in.Provider + out.ServerIP = in.ServerIP + out.StubDomains = in.StubDomains + out.UpstreamNameservers = in.UpstreamNameservers return nil } @@ -2109,13 +2111,15 @@ func Convert_v1alpha1_KubeDNSConfig_To_kops_KubeDNSConfig(in *KubeDNSConfig, out } func autoConvert_kops_KubeDNSConfig_To_v1alpha1_KubeDNSConfig(in *kops.KubeDNSConfig, out *KubeDNSConfig, s conversion.Scope) error { - out.Image = in.Image - out.Replicas = in.Replicas - out.Domain = in.Domain - out.ServerIP = in.ServerIP out.CacheMaxSize = in.CacheMaxSize out.CacheMaxConcurrent = in.CacheMaxConcurrent + out.Domain = in.Domain + out.Image = in.Image + out.Replicas = in.Replicas out.Provider = in.Provider + out.ServerIP = in.ServerIP + out.StubDomains = in.StubDomains + out.UpstreamNameservers = in.UpstreamNameservers return nil } diff --git a/pkg/apis/kops/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/kops/v1alpha1/zz_generated.deepcopy.go index 1239520626..c30c357b3d 100644 --- a/pkg/apis/kops/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/kops/v1alpha1/zz_generated.deepcopy.go @@ -706,7 +706,7 @@ func (in *ClusterSpec) DeepCopyInto(out *ClusterSpec) { *out = nil } else { *out = new(KubeDNSConfig) - **out = **in + (*in).DeepCopyInto(*out) } } if in.KubeAPIServer != nil { @@ -2107,6 +2107,23 @@ func (in *KubeControllerManagerConfig) DeepCopy() *KubeControllerManagerConfig { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *KubeDNSConfig) DeepCopyInto(out *KubeDNSConfig) { *out = *in + if in.StubDomains != nil { + in, out := &in.StubDomains, &out.StubDomains + *out = make(map[string][]string, len(*in)) + for key, val := range *in { + if val == nil { + (*out)[key] = nil + } else { + (*out)[key] = make([]string, len(val)) + copy((*out)[key], val) + } + } + } + if in.UpstreamNameservers != nil { + in, out := &in.UpstreamNameservers, &out.UpstreamNameservers + *out = make([]string, len(*in)) + copy(*out, *in) + } return } diff --git a/pkg/apis/kops/v1alpha2/cluster.go b/pkg/apis/kops/v1alpha2/cluster.go index c517ab0a8c..9c6121b232 100644 --- a/pkg/apis/kops/v1alpha2/cluster.go +++ b/pkg/apis/kops/v1alpha2/cluster.go @@ -284,21 +284,26 @@ type LoadBalancerAccessSpec struct { AdditionalSecurityGroups []string `json:"additionalSecurityGroups,omitempty"` } +// KubeDNSConfig defines the kube dns configuration type KubeDNSConfig struct { - // Image is the name of the docker image to run - // Deprecated as this is now in the addon - Image string `json:"image,omitempty"` - - // Deprecated as this is now in the addon, and controlled by autoscaler - Replicas int `json:"replicas,omitempty"` - Domain string `json:"domain,omitempty"` - ServerIP string `json:"serverIP,omitempty"` // CacheMaxSize is the maximum entries to keep in dnsmaq CacheMaxSize int `json:"cacheMaxSize,omitempty"` // CacheMaxConcurrent is the maximum number of concurrent queries for dnsmasq CacheMaxConcurrent int `json:"cacheMaxConcurrent,omitempty"` + // Domain is the dns domain + Domain string `json:"domain,omitempty"` + // Image is the name of the docker image to run - @deprecated as this is now in the addon + Image string `json:"image,omitempty"` + // Replicas is the number of pod replicas - @deprecated as this is now in the addon, and controlled by autoscaler + Replicas int `json:"replicas,omitempty"` // Provider indicates whether CoreDNS or kube-dns will be the default service discovery. Provider string `json:"provider,omitempty"` + // ServerIP is the server ip + ServerIP string `json:"serverIP,omitempty"` + // StubDomains redirects a domains to another DNS service + StubDomains map[string][]string `json:"stubDomains,omitempty"` + // UpstreamNameservers sets the upstream nameservers for queries not on the cluster domain + UpstreamNameservers []string `json:"upstreamNameservers,omitempty"` } // ExternalDNSConfig are options of the dns-controller diff --git a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go index afebef5150..021c4d45e6 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go @@ -2357,13 +2357,15 @@ func Convert_kops_KubeControllerManagerConfig_To_v1alpha2_KubeControllerManagerC } func autoConvert_v1alpha2_KubeDNSConfig_To_kops_KubeDNSConfig(in *KubeDNSConfig, out *kops.KubeDNSConfig, s conversion.Scope) error { - out.Image = in.Image - out.Replicas = in.Replicas - out.Domain = in.Domain - out.ServerIP = in.ServerIP out.CacheMaxSize = in.CacheMaxSize out.CacheMaxConcurrent = in.CacheMaxConcurrent + out.Domain = in.Domain + out.Image = in.Image + out.Replicas = in.Replicas out.Provider = in.Provider + out.ServerIP = in.ServerIP + out.StubDomains = in.StubDomains + out.UpstreamNameservers = in.UpstreamNameservers return nil } @@ -2373,13 +2375,15 @@ func Convert_v1alpha2_KubeDNSConfig_To_kops_KubeDNSConfig(in *KubeDNSConfig, out } func autoConvert_kops_KubeDNSConfig_To_v1alpha2_KubeDNSConfig(in *kops.KubeDNSConfig, out *KubeDNSConfig, s conversion.Scope) error { - out.Image = in.Image - out.Replicas = in.Replicas - out.Domain = in.Domain - out.ServerIP = in.ServerIP out.CacheMaxSize = in.CacheMaxSize out.CacheMaxConcurrent = in.CacheMaxConcurrent + out.Domain = in.Domain + out.Image = in.Image + out.Replicas = in.Replicas out.Provider = in.Provider + out.ServerIP = in.ServerIP + out.StubDomains = in.StubDomains + out.UpstreamNameservers = in.UpstreamNameservers return nil } diff --git a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go index ba7aef667b..8cd1994b5f 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go @@ -673,7 +673,7 @@ func (in *ClusterSpec) DeepCopyInto(out *ClusterSpec) { *out = nil } else { *out = new(KubeDNSConfig) - **out = **in + (*in).DeepCopyInto(*out) } } if in.KubeAPIServer != nil { @@ -2188,6 +2188,23 @@ func (in *KubeControllerManagerConfig) DeepCopy() *KubeControllerManagerConfig { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *KubeDNSConfig) DeepCopyInto(out *KubeDNSConfig) { *out = *in + if in.StubDomains != nil { + in, out := &in.StubDomains, &out.StubDomains + *out = make(map[string][]string, len(*in)) + for key, val := range *in { + if val == nil { + (*out)[key] = nil + } else { + (*out)[key] = make([]string, len(val)) + copy((*out)[key], val) + } + } + } + if in.UpstreamNameservers != nil { + in, out := &in.UpstreamNameservers, &out.UpstreamNameservers + *out = make([]string, len(*in)) + copy(*out, *in) + } return } diff --git a/pkg/apis/kops/validation/legacy.go b/pkg/apis/kops/validation/legacy.go index 53e5ff361c..848c871686 100644 --- a/pkg/apis/kops/validation/legacy.go +++ b/pkg/apis/kops/validation/legacy.go @@ -267,27 +267,44 @@ func ValidateCluster(c *kops.Cluster, strict bool) *field.Error { } } - // Check KubeDNS.ServerIP + // @check the custom kubedns options are valid if c.Spec.KubeDNS != nil { - serverIPString := c.Spec.KubeDNS.ServerIP - if serverIPString == "" { - return field.Required(fieldSpec.Child("KubeDNS", "ServerIP"), "Cluster did not have KubeDNS.ServerIP set") + if c.Spec.KubeDNS.ServerIP != "" { + address := c.Spec.KubeDNS.ServerIP + ip := net.ParseIP(address) + if ip == nil { + return field.Invalid(fieldSpec.Child("kubeDNS", "serverIP"), address, "Cluster had an invalid kubeDNS.serverIP") + } + if !serviceClusterIPRange.Contains(ip) { + return field.Invalid(fieldSpec.Child("kubeDNS", "serverIP"), address, fmt.Sprintf("ServiceClusterIPRange %q must contain the DNS Server IP %q", c.Spec.ServiceClusterIPRange, address)) + } + if c.Spec.Kubelet != nil && c.Spec.Kubelet.ClusterDNS != c.Spec.KubeDNS.ServerIP { + return field.Invalid(fieldSpec.Child("kubeDNS", "serverIP"), address, "Kubelet ClusterDNS did not match cluster kubeDNS.serverIP") + } + if c.Spec.MasterKubelet != nil && c.Spec.MasterKubelet.ClusterDNS != c.Spec.KubeDNS.ServerIP { + return field.Invalid(fieldSpec.Child("kubeDNS", "serverIP"), address, "MasterKubelet ClusterDNS did not match cluster kubeDNS.serverIP") + } } - dnsServiceIP := net.ParseIP(serverIPString) - if dnsServiceIP == nil { - return field.Invalid(fieldSpec.Child("KubeDNS", "ServerIP"), serverIPString, "Cluster had an invalid KubeDNS.ServerIP") + // @check the nameservers are valid + for i, x := range c.Spec.KubeDNS.UpstreamNameservers { + if ip := net.ParseIP(x); ip == nil { + return field.Invalid(fieldSpec.Child("kubeDNS", "upstreamNameservers").Index(i), x, "Invalid nameserver given, should be a valid ip address") + } } - if !serviceClusterIPRange.Contains(dnsServiceIP) { - return field.Invalid(fieldSpec.Child("KubeDNS", "ServerIP"), serverIPString, fmt.Sprintf("ServiceClusterIPRange %q must contain the DNS Server IP %q", c.Spec.ServiceClusterIPRange, serverIPString)) - } - - if c.Spec.Kubelet != nil && c.Spec.Kubelet.ClusterDNS != c.Spec.KubeDNS.ServerIP { - return field.Invalid(fieldSpec.Child("KubeDNS", "ServerIP"), serverIPString, "Kubelet ClusterDNS did not match cluster KubeDNS.ServerIP") - } - if c.Spec.MasterKubelet != nil && c.Spec.MasterKubelet.ClusterDNS != c.Spec.KubeDNS.ServerIP { - return field.Invalid(fieldSpec.Child("KubeDNS", "ServerIP"), serverIPString, "MasterKubelet ClusterDNS did not match cluster KubeDNS.ServerIP") + // @check the stubdomain if any + if c.Spec.KubeDNS.StubDomains != nil { + for domain, nameservers := range c.Spec.KubeDNS.StubDomains { + if len(nameservers) <= 0 { + return field.Invalid(fieldSpec.Child("kubeDNS", "stubDomains").Key(domain), domain, "No nameservers specified for the stub domain") + } + for i, x := range nameservers { + if ip := net.ParseIP(x); ip == nil { + return field.Invalid(fieldSpec.Child("kubeDNS", "stubDomains").Key(domain).Index(i), x, "Invalid nameserver given, should be a valid ip address") + } + } + } } } diff --git a/pkg/apis/kops/zz_generated.deepcopy.go b/pkg/apis/kops/zz_generated.deepcopy.go index 43cb44a532..18e271d4ac 100644 --- a/pkg/apis/kops/zz_generated.deepcopy.go +++ b/pkg/apis/kops/zz_generated.deepcopy.go @@ -778,7 +778,7 @@ func (in *ClusterSpec) DeepCopyInto(out *ClusterSpec) { *out = nil } else { *out = new(KubeDNSConfig) - **out = **in + (*in).DeepCopyInto(*out) } } if in.KubeAPIServer != nil { @@ -2367,6 +2367,23 @@ func (in *KubeControllerManagerConfig) DeepCopy() *KubeControllerManagerConfig { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *KubeDNSConfig) DeepCopyInto(out *KubeDNSConfig) { *out = *in + if in.StubDomains != nil { + in, out := &in.StubDomains, &out.StubDomains + *out = make(map[string][]string, len(*in)) + for key, val := range *in { + if val == nil { + (*out)[key] = nil + } else { + (*out)[key] = make([]string, len(val)) + copy((*out)[key], val) + } + } + } + if in.UpstreamNameservers != nil { + in, out := &in.UpstreamNameservers, &out.UpstreamNameservers + *out = make([]string, len(*in)) + copy(*out, *in) + } return } diff --git a/pkg/model/components/kubedns.go b/pkg/model/components/kubedns.go index 494bf7837c..dbb5c7743e 100644 --- a/pkg/model/components/kubedns.go +++ b/pkg/model/components/kubedns.go @@ -28,6 +28,7 @@ type KubeDnsOptionsBuilder struct { var _ loader.OptionsBuilder = &KubeDnsOptionsBuilder{} +// BuildOptions fills in the kubedns model func (b *KubeDnsOptionsBuilder) BuildOptions(o interface{}) error { clusterSpec := o.(*kops.ClusterSpec) @@ -45,12 +46,17 @@ func (b *KubeDnsOptionsBuilder) BuildOptions(o interface{}) error { clusterSpec.KubeDNS.CacheMaxConcurrent = 150 } - ip, err := WellKnownServiceIP(clusterSpec, 10) - if err != nil { - return err + if clusterSpec.KubeDNS.ServerIP == "" { + ip, err := WellKnownServiceIP(clusterSpec, 10) + if err != nil { + return err + } + clusterSpec.KubeDNS.ServerIP = ip.String() + } + + if clusterSpec.KubeDNS.Domain == "" { + clusterSpec.KubeDNS.Domain = clusterSpec.ClusterDNSDomain } - clusterSpec.KubeDNS.ServerIP = ip.String() - clusterSpec.KubeDNS.Domain = clusterSpec.ClusterDNSDomain return nil } diff --git a/upup/models/cloudup/resources/addons/kube-dns.addons.k8s.io/k8s-1.6.yaml.template b/upup/models/cloudup/resources/addons/kube-dns.addons.k8s.io/k8s-1.6.yaml.template index 5e5c3b831c..ddbdf3b2db 100644 --- a/upup/models/cloudup/resources/addons/kube-dns.addons.k8s.io/k8s-1.6.yaml.template +++ b/upup/models/cloudup/resources/addons/kube-dns.addons.k8s.io/k8s-1.6.yaml.template @@ -12,6 +12,26 @@ # See the License for the specific language governing permissions and # limitations under the License. +{{- if or (.KubeDNS.UpstreamNameservers) (.KubeDNS.StubDomains) }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: kube-dns + namespace: kube-system +data: + {{- if .KubeDNS.UpstreamNameservers }} + upstreamNameservers: | + {{ ToJSON .KubeDNS.UpstreamNameservers }} + {{- end }} + {{- if .KubeDNS.StubDomains }} + stubDomains: | + {{ ToJSON .KubeDNS.StubDomains }} + {{- end }} +{{- end }} + +--- + apiVersion: extensions/v1beta1 kind: Deployment metadata: @@ -126,9 +146,9 @@ spec: initialDelaySeconds: 3 timeoutSeconds: 5 args: - - --domain={{ KubeDNS.Domain }}. - - --dns-port=10053 - --config-dir=/kube-dns-config + - --dns-port=10053 + - --domain={{ KubeDNS.Domain }}. - --v=2 env: - name: PROMETHEUS_PORT diff --git a/upup/pkg/fi/cloudup/template_functions.go b/upup/pkg/fi/cloudup/template_functions.go index 6ec0ad6f91..77baeb1466 100644 --- a/upup/pkg/fi/cloudup/template_functions.go +++ b/upup/pkg/fi/cloudup/template_functions.go @@ -28,6 +28,7 @@ When defining a new function: package cloudup import ( + "encoding/json" "fmt" "os" "strconv" @@ -59,6 +60,7 @@ func (tf *TemplateFunctions) AddTo(dest template.FuncMap) { dest["EtcdScheme"] = tf.EtcdScheme dest["SharedVPC"] = tf.SharedVPC dest["UseEtcdTLS"] = tf.UseEtcdTLS + dest["ToJSON"] = tf.ToJSON // Remember that we may be on a different arch from the target. Hard-code for now. dest["Arch"] = func() string { return "amd64" } dest["replace"] = func(s, find, replace string) string { @@ -119,6 +121,16 @@ func (tf *TemplateFunctions) UseEtcdTLS() bool { return false } +// ToJSON returns a json representation of the struct or on error an empty string +func (tf *TemplateFunctions) ToJSON(data interface{}) string { + encoded, err := json.Marshal(data) + if err != nil { + return "" + } + + return string(encoded) +} + // EtcdScheme parses and grabs the protocol to the etcd cluster func (tf *TemplateFunctions) EtcdScheme() string { if tf.UseEtcdTLS() {