Merge pull request #12792 from justinsb/gossip_coredns_hosts_via_services

gossip: support resolution of k8s.local names from pods (via services)

cmd/kops-controller/BUILD.bazel

@@ -21,6 +21,7 @@ go_library(
         "//vendor/k8s.io/api/coordination/v1:go_default_library",
         "//vendor/k8s.io/api/core/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
         "//vendor/k8s.io/client-go/plugin/pkg/client/auth/gcp:go_default_library",
         "//vendor/k8s.io/klog/v2:go_default_library",
         "//vendor/k8s.io/klog/v2/klogr:go_default_library",

cmd/kops-controller/controllers/BUILD.bazel

@@ -4,6 +4,7 @@ go_library(
     name = "go_default_library",
     srcs = [
         "awsipam.go",
+        "hosts_controller.go",
         "legacy_node_controller.go",
         "node_controller.go",
     ],
@@ -27,7 +28,9 @@ go_library(
         "//vendor/k8s.io/api/core/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
+        "//vendor/k8s.io/client-go/dynamic:go_default_library",
         "//vendor/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library",
         "//vendor/k8s.io/klog/v2:go_default_library",
         "//vendor/sigs.k8s.io/controller-runtime:go_default_library",

cmd/kops-controller/controllers/hosts_controller.go

@@ -0,0 +1,183 @@
+/*
+Copyright 2021 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controllers
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"reflect"
+	"sort"
+	"strings"
+
+	"github.com/go-logr/logr"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/dynamic"
+	"k8s.io/klog/v2"
+	"k8s.io/kops/pkg/apis/kops"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/manager"
+)
+
+// HostsReconciler populates an /etc/hosts style file in the CoreDNS config map,
+// supporting in-pod resolution of our k8s.local entries.
+type HostsReconciler struct {
+	// configMapID identifies the configmap we should update
+	configMapID types.NamespacedName
+
+	// client is the controller-runtime client
+	client client.Client
+
+	// log is a logr
+	log logr.Logger
+
+	// dynamicClient is a client-go client for patching ConfigMaps
+	dynamicClient dynamic.Interface
+
+	// lastUpdate holds the last value we updated, to reduce spurious updates.
+	lastUpdate *managedConfigMap
+}
+
+// NewHostsReconciler is the constructor for a HostsReconciler
+func NewHostsReconciler(mgr manager.Manager, configMapID types.NamespacedName) (*HostsReconciler, error) {
+	r := &HostsReconciler{
+		client:      mgr.GetClient(),
+		log:         ctrl.Log.WithName("controllers").WithName("Hosts"),
+		configMapID: configMapID,
+	}
+
+	dynamicClient, err := dynamic.NewForConfig(mgr.GetConfig())
+	if err != nil {
+		return nil, fmt.Errorf("error building dynamic client: %v", err)
+	}
+	r.dynamicClient = dynamicClient
+
+	return r, nil
+}
+
+// +kubebuilder:rbac:groups=,resources=endpoints,verbs=get;list;watch
+
+// +kubebuilder:rbac:groups=,resources=configmaps,namespace=kube-system,resourceNames=coredns,verbs=get;patch
+
+// Reconcile is the main reconciler function that observes node changes.
+func (r *HostsReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	_ = r.log.WithValues("endpoints", req.NamespacedName)
+
+	// Although we label the service, the labels get copied to the endpoints by the kube-controller-manager.
+	endpointsLabels := client.HasLabels([]string{kops.DiscoveryLabelKey})
+
+	endpointsList := &corev1.EndpointsList{}
+
+	// For security, we only process endpoints in kube-system
+	if err := r.client.List(ctx, endpointsList, endpointsLabels, client.InNamespace("kube-system")); err != nil {
+		klog.Warningf("unable to list endpoints: %v", err)
+		return ctrl.Result{}, err
+	}
+
+	return ctrl.Result{}, r.updateHosts(ctx, endpointsList)
+}
+
+func (r *HostsReconciler) updateHosts(ctx context.Context, endpointsList *corev1.EndpointsList) error {
+	addrToHosts := make(map[string][]string)
+
+	for i := range endpointsList.Items {
+		endpoints := &endpointsList.Items[i]
+
+		hostname := endpoints.Labels[kops.DiscoveryLabelKey]
+		if hostname == "" {
+			klog.Warningf("endpoints %s/%s found without discovery label %q; filtering is not working correctly", endpoints.Name, endpoints.Namespace, kops.DiscoveryLabelKey)
+			continue
+		}
+
+		for j := range endpoints.Subsets {
+			subset := &endpoints.Subsets[j]
+
+			for k := range subset.Addresses {
+				address := &subset.Addresses[k]
+
+				ip := address.IP
+				if ip != "" {
+					addrToHosts[ip] = append(addrToHosts[ip], hostname)
+				}
+			}
+		}
+	}
+
+	return r.updateConfigMap(ctx, addrToHosts)
+}
+
+// managedConfigMap holds the fields we manage
+type managedConfigMap struct {
+	APIVersion string            `json:"apiVersion"`
+	Kind       string            `json:"kind"`
+	Data       map[string]string `json:"data"`
+}
+
+func (r *HostsReconciler) updateConfigMap(ctx context.Context, addrToHosts map[string][]string) error {
+	var block []string
+	for addr, hosts := range addrToHosts {
+		sort.Strings(hosts)
+		block = append(block, addr+"\t"+strings.Join(hosts, " "))
+	}
+	// Sort into a consistent order to minimize updates
+	sort.Strings(block)
+
+	hosts := strings.Join(block, "\n")
+
+	data := &managedConfigMap{}
+	data.APIVersion = "v1"
+	data.Kind = "ConfigMap"
+	data.Data = map[string]string{"hosts": hosts}
+
+	if r.lastUpdate != nil && reflect.DeepEqual(r.lastUpdate, data) {
+		klog.Infof("skipping hosts configmap update (unchanged): %#v", data)
+		return nil
+	}
+
+	klog.Infof("patching hosts configmap: %#v", data)
+
+	configmapGVR := schema.GroupVersionResource{Group: "", Version: "v1", Resource: "configmaps"}
+
+	patch, err := json.Marshal(data)
+	if err != nil {
+		return fmt.Errorf("failed to marshal patch: %w", err)
+	}
+
+	// It is strongly recommended for controllers to always "force" conflicts, since they might not be able to resolve or act on these conflicts.
+	force := true
+	patchOpts := metav1.PatchOptions{
+		FieldManager: "kops-controller.kops.k8s.io/hosts",
+		Force:        &force,
+	}
+	if _, err := r.dynamicClient.Resource(configmapGVR).Namespace(r.configMapID.Namespace).Patch(ctx, r.configMapID.Name, types.ApplyPatchType, patch, patchOpts); err != nil {
+		return fmt.Errorf("failed to patch configmap: %w", err)
+	}
+
+	r.lastUpdate = data
+
+	return nil
+}
+
+func (r *HostsReconciler) SetupWithManager(mgr ctrl.Manager) error {
+	return ctrl.NewControllerManagedBy(mgr).
+		For(&corev1.Endpoints{}).
+		Complete(r)
+}

cmd/kops-controller/main.go

@@ -24,6 +24,7 @@ import (
 	coordinationv1 "k8s.io/api/coordination/v1"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
 	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
 	"k8s.io/klog/v2"
 	"k8s.io/klog/v2/klogr"
@@ -155,6 +156,11 @@ func main() {
 		os.Exit(1)
 	}
 
+	if err := addGossipController(mgr, &opt); err != nil {
+		setupLog.Error(err, "unable to create controller", "controller", "GossipController")
+		os.Exit(1)
+	}
+
 	// +kubebuilder:scaffold:builder
 
 	setupLog.Info("starting manager")
@@ -241,3 +247,25 @@ func addNodeController(mgr manager.Manager, opt *config.Options) error {
 
 	return nil
 }
+
+func addGossipController(mgr manager.Manager, opt *config.Options) error {
+	if opt.Discovery == nil || !opt.Discovery.Enabled {
+		return nil
+	}
+
+	configMapID := types.NamespacedName{
+		Namespace: "kube-system",
+		Name:      "coredns",
+	}
+
+	controller, err := controllers.NewHostsReconciler(mgr, configMapID)
+	if err != nil {
+		return err
+	}
+
+	if err := controller.SetupWithManager(mgr); err != nil {
+		return err
+	}
+
+	return nil
+}

cmd/kops-controller/pkg/config/options.go

@@ -29,6 +29,9 @@ type Options struct {
 
 	// EnableCloudIPAM enables the cloud IPAM controller.
 	EnableCloudIPAM bool `json:"enableCloudIPAM,omitempty"`
+
+	// Discovery configures options relating to discovery, particularly for gossip mode.
+	Discovery *DiscoveryOptions `json:"discovery,omitempty"`
 }
 
 func (o *Options) PopulateDefaults() {
@@ -58,3 +61,9 @@ type ServerProviderOptions struct {
 	AWS *awsup.AWSVerifierOptions  `json:"aws,omitempty"`
 	GCE *gcetpm.TPMVerifierOptions `json:"gce,omitempty"`
 }
+
+// DiscoveryOptions configures our support for discovery, particularly gossip DNS (i.e. k8s.local)
+type DiscoveryOptions struct {
+	// Enabled specifies whether support for discovery population is enabled.
+	Enabled bool `json:"enabled"`
+}

go.mod

@@ -80,6 +80,7 @@ require (
 	gopkg.in/gcfg.v1 v1.2.3
 	gopkg.in/inf.v0 v0.9.1
 	gopkg.in/square/go-jose.v2 v2.5.1
+	gopkg.in/yaml.v2 v2.4.0
 	helm.sh/helm/v3 v3.7.1
 	k8s.io/api v0.22.2
 	k8s.io/apimachinery v0.22.2
@@ -225,7 +226,6 @@ require (
 	google.golang.org/protobuf v1.27.1 // indirect
 	gopkg.in/ini.v1 v1.62.0 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
-	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
 	k8s.io/apiextensions-apiserver v0.22.2 // indirect
 	k8s.io/cloud-provider v0.22.2 // indirect

pkg/apis/kops/labels.go

@@ -28,4 +28,8 @@ const (
 
 	// UpdatePolicyExternal is a value for ClusterSpec.UpdatePolicy and InstanceGroup.UpdatePolicy indicating that upgrades are done externally, and we should disable automatic upgrades
 	UpdatePolicyExternal = "external"
+
+	// DiscoveryLabelKey is the label we use for services that should be exposed internally.
+	// Endpoints get the same labels as their services.
+	DiscoveryLabelKey = "discovery.kops.k8s.io/internal-name"
 )

pkg/kubemanifest/BUILD.bazel

@@ -9,13 +9,16 @@ go_library(
         "priority.go",
         "visitor.go",
         "volumes.go",
+        "yaml.go",
     ],
     importpath = "k8s.io/kops/pkg/kubemanifest",
     visibility = ["//visibility:public"],
     deps = [
         "//util/pkg/text:go_default_library",
+        "//vendor/gopkg.in/yaml.v2:go_default_library",
         "//vendor/k8s.io/api/core/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
         "//vendor/k8s.io/klog/v2:go_default_library",
         "//vendor/sigs.k8s.io/yaml:go_default_library",
     ],

pkg/kubemanifest/yaml.go

@@ -0,0 +1,69 @@
+/*
+Copyright 2021 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kubemanifest
+
+import (
+	"encoding/json"
+	"fmt"
+
+	yamlv2 "gopkg.in/yaml.v2"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/klog/v2"
+)
+
+// KubeObjectToApplyYAML returns the kubernetes object converted to YAML, with "noisy" fields removed.
+//
+// We remove:
+//  * status (can't be applied, shouldn't be specified)
+//  * metadata.creationTimestamp (can't be applied, shouldn't be specified)
+func KubeObjectToApplyYAML(data runtime.Object) (string, error) {
+	// This logic is inlined sigs.k8s.io/yaml.Marshal, but we delete some fields in the middle.
+
+	// Convert the object to JSON bytes
+	j, err := json.Marshal(data)
+	if err != nil {
+		return "", fmt.Errorf("error marshaling into JSON: %v", err)
+	}
+
+	// Convert the JSON to a map.
+	jsonObj := make(map[string]interface{})
+	if err := yamlv2.Unmarshal(j, &jsonObj); err != nil {
+		return "", err
+	}
+
+	// Remove status (can't be applied, shouldn't be specified)
+	delete(jsonObj, "status")
+
+	// Remove metadata.creationTimestamp (can't be applied, shouldn't be specified)
+	metadataObj, found := jsonObj["metadata"]
+	if found {
+		if metadata, ok := metadataObj.(map[interface{}]interface{}); ok {
+			delete(metadata, "creationTimestamp")
+		} else {
+			klog.Warningf("unexpected type for object metadata: %T", metadataObj)
+		}
+	} else {
+		klog.Warningf("object did not have metadata: %#v", jsonObj)
+	}
+
+	// Marshal the cleaned-up map into YAML.
+	y, err := yamlv2.Marshal(jsonObj)
+	if err != nil {
+		return "", err
+	}
+	return string(y), nil
+}

pkg/model/components/etcdmanager/model.go

@@ -265,10 +265,8 @@ func (b *EtcdManagerBuilder) buildPod(etcdCluster kops.EtcdClusterSpec) (*v1.Pod
 	} else {
 		clientHost = "__name__"
 	}
-	clientPort := 4001
 
 	clusterName := "etcd-" + etcdCluster.Name
-	peerPort := 2380
 	backupStore := ""
 	if etcdCluster.Backups != nil {
 		backupStore = etcdCluster.Backups.BackupStore
@@ -287,12 +285,9 @@ func (b *EtcdManagerBuilder) buildPod(etcdCluster kops.EtcdClusterSpec) (*v1.Pod
 	if pod.Labels == nil {
 		pod.Labels = make(map[string]string)
 	}
-	pod.Labels["k8s-app"] = pod.Name
-
-	// TODO: Use a socket file for the quarantine port
-	quarantinedClientPort := wellknownports.EtcdMainQuarantinedClientPort
-
-	grpcPort := wellknownports.EtcdMainGRPC
+	for k, v := range SelectorForCluster(etcdCluster) {
+		pod.Labels[k] = v
+	}
 
 	// The dns suffix logic mirrors the existing logic, so we should be compatible with existing clusters
 	// (etcd makes it difficult to change peer urls, treating it as a cluster event, for reasons unknown)
@@ -307,20 +302,19 @@ func (b *EtcdManagerBuilder) buildPod(etcdCluster kops.EtcdClusterSpec) (*v1.Pod
 		dnsInternalSuffix = ".internal." + b.Cluster.ObjectMeta.Name
 	}
 
+	ports, err := PortsForCluster(etcdCluster)
+	if err != nil {
+		return nil, err
+	}
+
 	switch etcdCluster.Name {
 	case "main":
 		clusterName = "etcd"
 
 	case "events":
-		clientPort = 4002
-		peerPort = 2381
-		grpcPort = wellknownports.EtcdEventsGRPC
-		quarantinedClientPort = wellknownports.EtcdEventsQuarantinedClientPort
+		// ok
+
 	case "cilium":
-		clientPort = 4003
-		peerPort = 2382
-		grpcPort = wellknownports.EtcdCiliumGRPC
-		quarantinedClientPort = wellknownports.EtcdCiliumQuarantinedClientPort
 		if !featureflag.APIServerNodes.Enabled() {
 			clientHost = b.Cluster.Spec.MasterInternalName
 		}
@@ -343,7 +337,7 @@ func (b *EtcdManagerBuilder) buildPod(etcdCluster kops.EtcdClusterSpec) (*v1.Pod
 		Containerized: true,
 		ClusterName:   clusterName,
 		BackupStore:   backupStore,
-		GrpcPort:      grpcPort,
+		GrpcPort:      ports.GRPCPort,
 		DNSSuffix:     dnsInternalSuffix,
 	}
 
@@ -361,9 +355,9 @@ func (b *EtcdManagerBuilder) buildPod(etcdCluster kops.EtcdClusterSpec) (*v1.Pod
 	{
 		scheme := "https"
 
-		config.PeerUrls = fmt.Sprintf("%s://__name__:%d", scheme, peerPort)
-		config.ClientUrls = fmt.Sprintf("%s://%s:%d", scheme, clientHost, clientPort)
-		config.QuarantineClientUrls = fmt.Sprintf("%s://__name__:%d", scheme, quarantinedClientPort)
+		config.PeerUrls = fmt.Sprintf("%s://__name__:%d", scheme, ports.PeerPort)
+		config.ClientUrls = fmt.Sprintf("%s://%s:%d", scheme, clientHost, ports.ClientPort)
+		config.QuarantineClientUrls = fmt.Sprintf("%s://__name__:%d", scheme, ports.QuarantinedGRPCPort)
 
 		// TODO: We need to wire these into the etcd-manager spec
 		// // add timeout/heartbeat settings
@@ -573,3 +567,50 @@ type config struct {
 	VolumeNameTag         string   `flag:"volume-name-tag"`
 	DNSSuffix             string   `flag:"dns-suffix"`
 }
+
+// SelectorForCluster returns the selector that should be used to select our pods (from services)
+func SelectorForCluster(etcdCluster kops.EtcdClusterSpec) map[string]string {
+	return map[string]string{
+		"k8s-app": "etcd-manager-" + etcdCluster.Name,
+	}
+}
+
+type Ports struct {
+	ClientPort          int
+	PeerPort            int
+	GRPCPort            int
+	QuarantinedGRPCPort int
+}
+
+// PortsForCluster returns the ports that the cluster users.
+func PortsForCluster(etcdCluster kops.EtcdClusterSpec) (Ports, error) {
+	switch etcdCluster.Name {
+	case "main":
+		return Ports{
+			GRPCPort: wellknownports.EtcdMainGRPC,
+			// TODO: Use a socket file for the quarantine port
+			QuarantinedGRPCPort: wellknownports.EtcdMainQuarantinedClientPort,
+			ClientPort:          4001,
+			PeerPort:            2380,
+		}, nil
+
+	case "events":
+		return Ports{
+			GRPCPort:            wellknownports.EtcdEventsGRPC,
+			QuarantinedGRPCPort: wellknownports.EtcdEventsQuarantinedClientPort,
+			ClientPort:          4002,
+			PeerPort:            2381,
+		}, nil
+	case "cilium":
+		return Ports{
+			GRPCPort:            wellknownports.EtcdCiliumGRPC,
+			QuarantinedGRPCPort: wellknownports.EtcdCiliumQuarantinedClientPort,
+			ClientPort:          4003,
+			PeerPort:            2382,
+		}, nil
+
+	default:
+		return Ports{}, fmt.Errorf("unknown etcd cluster key %q", etcdCluster.Name)
+	}
+
+}

pkg/model/components/kopscontroller/BUILD.bazel

@@ -0,0 +1,18 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["template_functions.go"],
+    importpath = "k8s.io/kops/pkg/model/components/kopscontroller",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//pkg/apis/kops:go_default_library",
+        "//pkg/dns:go_default_library",
+        "//pkg/featureflag:go_default_library",
+        "//pkg/model/components/etcdmanager:go_default_library",
+        "//pkg/wellknownports:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/intstr:go_default_library",
+    ],
+)

pkg/model/components/kopscontroller/template_functions.go

@@ -0,0 +1,132 @@
+/*
+Copyright 2021 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kopscontroller
+
+import (
+	"text/template"
+
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/intstr"
+	"k8s.io/kops/pkg/apis/kops"
+	"k8s.io/kops/pkg/dns"
+	"k8s.io/kops/pkg/featureflag"
+	"k8s.io/kops/pkg/model/components/etcdmanager"
+	"k8s.io/kops/pkg/wellknownports"
+)
+
+// AddTemplateFunctions registers template functions for KopsController
+func AddTemplateFunctions(cluster *kops.Cluster, dest template.FuncMap) {
+	t := &templateFunctions{
+		Cluster: cluster,
+	}
+	dest["KopsController"] = func() *templateFunctions {
+		return t
+	}
+}
+
+// templateFunctions implements the KopsController template object helper.
+type templateFunctions struct {
+	Cluster *kops.Cluster
+}
+
+// KopsControllerConfig returns the yaml configuration for kops-controller
+func (t *templateFunctions) GossipServices() ([]*corev1.Service, error) {
+	if !dns.IsGossipHostname(t.Cluster.Name) {
+		return nil, nil
+	}
+
+	suffix := t.Cluster.Name
+
+	var services []*corev1.Service
+
+	// api service
+	{
+		service := buildHeadlessService(types.NamespacedName{Name: "api-internal", Namespace: "kube-system"})
+		service.Spec.Ports = []corev1.ServicePort{
+			{Name: "https", Port: 443, Protocol: corev1.ProtocolTCP},
+		}
+		service.Spec.Selector = map[string]string{
+			"k8s-app": "kops-controller",
+		}
+		service.Labels = map[string]string{
+			kops.DiscoveryLabelKey: "api.internal." + suffix,
+		}
+		services = append(services, service)
+	}
+
+	// kops-controller service
+	{
+		service := buildHeadlessService(types.NamespacedName{Name: "kops-controller-internal", Namespace: "kube-system"})
+		service.Spec.Ports = []corev1.ServicePort{
+			{Name: "https", Port: wellknownports.KopsControllerPort, Protocol: corev1.ProtocolTCP},
+		}
+		service.Spec.Selector = map[string]string{
+			"k8s-app": "kops-controller",
+		}
+		service.Labels = map[string]string{
+			kops.DiscoveryLabelKey: "kops-controller.internal." + suffix,
+		}
+		services = append(services, service)
+	}
+
+	// etcd services
+	if featureflag.APIServerNodes.Enabled() {
+		for _, etcdCluster := range t.Cluster.Spec.EtcdClusters {
+			name := "etcd-" + etcdCluster.Name + "-internal"
+			service := buildHeadlessService(types.NamespacedName{Name: name, Namespace: "kube-system"})
+			ports, err := etcdmanager.PortsForCluster(etcdCluster)
+			if err != nil {
+				return nil, err
+			}
+			service.Spec.Ports = []corev1.ServicePort{
+				{Name: "https", Port: int32(ports.ClientPort), Protocol: corev1.ProtocolTCP},
+			}
+			service.Labels = map[string]string{
+				kops.DiscoveryLabelKey: etcdCluster.Name + ".etcd." + suffix,
+			}
+			service.Spec.Selector = etcdmanager.SelectorForCluster(etcdCluster)
+			services = append(services, service)
+		}
+	}
+
+	// We set the target port, to make applying cleaner
+	for _, service := range services {
+		for i := range service.Spec.Ports {
+			port := &service.Spec.Ports[i]
+			if port.TargetPort == intstr.FromInt(0) {
+				port.TargetPort = intstr.FromInt(int(port.Port))
+			}
+		}
+	}
+
+	return services, nil
+}
+
+// buildHeadlessService is a helper to build a headless service
+func buildHeadlessService(name types.NamespacedName) *corev1.Service {
+	s := &corev1.Service{}
+	s.APIVersion = "v1"
+	s.Kind = "Service"
+	s.Name = name.Name
+	s.Namespace = name.Namespace
+
+	s.Spec.ClusterIP = corev1.ClusterIPNone
+	s.Spec.Type = corev1.ServiceTypeClusterIP
+
+	return s
+}

tests/integration/update_cluster/aws-lb-controller/data/aws_s3_bucket_object_minimal.example.com-addons-bootstrap_content

@@ -20,7 +20,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.12
     manifest: coredns.addons.k8s.io/k8s-1.12.yaml
-    manifestHash: 88ffe1a3752cf290450cc94bd53aea49a665e411dbf4cfe9c1a2cc5b027f12ef
+    manifestHash: 12b67f439637253329bf6fb2ee23b3ef65959621720c863263c13da8271bff89
     name: coredns.addons.k8s.io
     selector:
       k8s-addon: coredns.addons.k8s.io

tests/integration/update_cluster/aws-lb-controller/data/aws_s3_bucket_object_minimal.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content

@@ -200,9 +200,6 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          items:
-          - key: Corefile
-            path: Corefile
           name: coredns
         name: config-volume
 

tests/integration/update_cluster/bastionadditional_user-data/data/aws_s3_bucket_object_bastionuserdata.example.com-addons-bootstrap_content

@@ -20,7 +20,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.12
     manifest: coredns.addons.k8s.io/k8s-1.12.yaml
-    manifestHash: 88ffe1a3752cf290450cc94bd53aea49a665e411dbf4cfe9c1a2cc5b027f12ef
+    manifestHash: 12b67f439637253329bf6fb2ee23b3ef65959621720c863263c13da8271bff89
     name: coredns.addons.k8s.io
     selector:
       k8s-addon: coredns.addons.k8s.io

tests/integration/update_cluster/bastionadditional_user-data/data/aws_s3_bucket_object_bastionuserdata.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content

@@ -200,9 +200,6 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          items:
-          - key: Corefile
-            path: Corefile
           name: coredns
         name: config-volume
 

tests/integration/update_cluster/complex/data/aws_s3_bucket_object_complex.example.com-addons-bootstrap_content

@@ -20,7 +20,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.12
     manifest: coredns.addons.k8s.io/k8s-1.12.yaml
-    manifestHash: 88ffe1a3752cf290450cc94bd53aea49a665e411dbf4cfe9c1a2cc5b027f12ef
+    manifestHash: 12b67f439637253329bf6fb2ee23b3ef65959621720c863263c13da8271bff89
     name: coredns.addons.k8s.io
     selector:
       k8s-addon: coredns.addons.k8s.io

tests/integration/update_cluster/complex/data/aws_s3_bucket_object_complex.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content

@@ -200,9 +200,6 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          items:
-          - key: Corefile
-            path: Corefile
           name: coredns
         name: config-volume
 

tests/integration/update_cluster/compress/data/aws_s3_bucket_object_compress.example.com-addons-bootstrap_content

@@ -20,7 +20,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.12
     manifest: coredns.addons.k8s.io/k8s-1.12.yaml
-    manifestHash: 88ffe1a3752cf290450cc94bd53aea49a665e411dbf4cfe9c1a2cc5b027f12ef
+    manifestHash: 12b67f439637253329bf6fb2ee23b3ef65959621720c863263c13da8271bff89
     name: coredns.addons.k8s.io
     selector:
       k8s-addon: coredns.addons.k8s.io

tests/integration/update_cluster/compress/data/aws_s3_bucket_object_compress.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content

@@ -200,9 +200,6 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          items:
-          - key: Corefile
-            path: Corefile
           name: coredns
         name: config-volume
 

tests/integration/update_cluster/digit/data/aws_s3_bucket_object_123.example.com-addons-bootstrap_content

@@ -20,7 +20,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.12
     manifest: coredns.addons.k8s.io/k8s-1.12.yaml
-    manifestHash: 88ffe1a3752cf290450cc94bd53aea49a665e411dbf4cfe9c1a2cc5b027f12ef
+    manifestHash: 12b67f439637253329bf6fb2ee23b3ef65959621720c863263c13da8271bff89
     name: coredns.addons.k8s.io
     selector:
       k8s-addon: coredns.addons.k8s.io

tests/integration/update_cluster/digit/data/aws_s3_bucket_object_123.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content

@@ -200,9 +200,6 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          items:
-          - key: Corefile
-            path: Corefile
           name: coredns
         name: config-volume
 

tests/integration/update_cluster/existing_iam/data/aws_s3_bucket_object_existing-iam.example.com-addons-bootstrap_content

@@ -20,7 +20,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.12
     manifest: coredns.addons.k8s.io/k8s-1.12.yaml
-    manifestHash: 88ffe1a3752cf290450cc94bd53aea49a665e411dbf4cfe9c1a2cc5b027f12ef
+    manifestHash: 12b67f439637253329bf6fb2ee23b3ef65959621720c863263c13da8271bff89
     name: coredns.addons.k8s.io
     selector:
       k8s-addon: coredns.addons.k8s.io

tests/integration/update_cluster/existing_iam/data/aws_s3_bucket_object_existing-iam.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content

@@ -200,9 +200,6 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          items:
-          - key: Corefile
-            path: Corefile
           name: coredns
         name: config-volume
 

tests/integration/update_cluster/existing_sg/data/aws_s3_bucket_object_existingsg.example.com-addons-bootstrap_content

@@ -20,7 +20,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.12
     manifest: coredns.addons.k8s.io/k8s-1.12.yaml
-    manifestHash: 88ffe1a3752cf290450cc94bd53aea49a665e411dbf4cfe9c1a2cc5b027f12ef
+    manifestHash: 12b67f439637253329bf6fb2ee23b3ef65959621720c863263c13da8271bff89
     name: coredns.addons.k8s.io
     selector:
       k8s-addon: coredns.addons.k8s.io

tests/integration/update_cluster/existing_sg/data/aws_s3_bucket_object_existingsg.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content

@@ -200,9 +200,6 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          items:
-          - key: Corefile
-            path: Corefile
           name: coredns
         name: config-volume
 

tests/integration/update_cluster/external_dns/data/aws_s3_bucket_object_minimal.example.com-addons-bootstrap_content

@@ -20,7 +20,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.12
     manifest: coredns.addons.k8s.io/k8s-1.12.yaml
-    manifestHash: 88ffe1a3752cf290450cc94bd53aea49a665e411dbf4cfe9c1a2cc5b027f12ef
+    manifestHash: 12b67f439637253329bf6fb2ee23b3ef65959621720c863263c13da8271bff89
     name: coredns.addons.k8s.io
     selector:
       k8s-addon: coredns.addons.k8s.io

tests/integration/update_cluster/external_dns/data/aws_s3_bucket_object_minimal.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content

@@ -200,9 +200,6 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          items:
-          - key: Corefile
-            path: Corefile
           name: coredns
         name: config-volume
 

tests/integration/update_cluster/external_dns_irsa/data/aws_s3_bucket_object_minimal.example.com-addons-bootstrap_content

@@ -20,7 +20,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.12
     manifest: coredns.addons.k8s.io/k8s-1.12.yaml
-    manifestHash: 88ffe1a3752cf290450cc94bd53aea49a665e411dbf4cfe9c1a2cc5b027f12ef
+    manifestHash: 12b67f439637253329bf6fb2ee23b3ef65959621720c863263c13da8271bff89
     name: coredns.addons.k8s.io
     selector:
       k8s-addon: coredns.addons.k8s.io

tests/integration/update_cluster/external_dns_irsa/data/aws_s3_bucket_object_minimal.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content

@@ -200,9 +200,6 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          items:
-          - key: Corefile
-            path: Corefile
           name: coredns
         name: config-volume
 

tests/integration/update_cluster/externallb/data/aws_s3_bucket_object_externallb.example.com-addons-bootstrap_content

@@ -20,7 +20,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.12
     manifest: coredns.addons.k8s.io/k8s-1.12.yaml
-    manifestHash: 88ffe1a3752cf290450cc94bd53aea49a665e411dbf4cfe9c1a2cc5b027f12ef
+    manifestHash: 12b67f439637253329bf6fb2ee23b3ef65959621720c863263c13da8271bff89
     name: coredns.addons.k8s.io
     selector:
       k8s-addon: coredns.addons.k8s.io

tests/integration/update_cluster/externallb/data/aws_s3_bucket_object_externallb.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content

@@ -200,9 +200,6 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          items:
-          - key: Corefile
-            path: Corefile
           name: coredns
         name: config-volume
 

tests/integration/update_cluster/externalpolicies/data/aws_s3_bucket_object_externalpolicies.example.com-addons-bootstrap_content

@@ -20,7 +20,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.12
     manifest: coredns.addons.k8s.io/k8s-1.12.yaml
-    manifestHash: 88ffe1a3752cf290450cc94bd53aea49a665e411dbf4cfe9c1a2cc5b027f12ef
+    manifestHash: 12b67f439637253329bf6fb2ee23b3ef65959621720c863263c13da8271bff89
     name: coredns.addons.k8s.io
     selector:
       k8s-addon: coredns.addons.k8s.io

tests/integration/update_cluster/externalpolicies/data/aws_s3_bucket_object_externalpolicies.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content

@@ -200,9 +200,6 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          items:
-          - key: Corefile
-            path: Corefile
           name: coredns
         name: config-volume
 

tests/integration/update_cluster/ha/data/aws_s3_bucket_object_ha.example.com-addons-bootstrap_content

@@ -20,7 +20,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.12
     manifest: coredns.addons.k8s.io/k8s-1.12.yaml
-    manifestHash: 88ffe1a3752cf290450cc94bd53aea49a665e411dbf4cfe9c1a2cc5b027f12ef
+    manifestHash: 12b67f439637253329bf6fb2ee23b3ef65959621720c863263c13da8271bff89
     name: coredns.addons.k8s.io
     selector:
       k8s-addon: coredns.addons.k8s.io

tests/integration/update_cluster/ha/data/aws_s3_bucket_object_ha.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content

@@ -200,9 +200,6 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          items:
-          - key: Corefile
-            path: Corefile
           name: coredns
         name: config-volume
 

tests/integration/update_cluster/ha_gce/data/aws_s3_bucket_object_ha-gce.example.com-addons-bootstrap_content

@@ -20,7 +20,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.12
     manifest: coredns.addons.k8s.io/k8s-1.12.yaml
-    manifestHash: 88ffe1a3752cf290450cc94bd53aea49a665e411dbf4cfe9c1a2cc5b027f12ef
+    manifestHash: 12b67f439637253329bf6fb2ee23b3ef65959621720c863263c13da8271bff89
     name: coredns.addons.k8s.io
     selector:
       k8s-addon: coredns.addons.k8s.io

tests/integration/update_cluster/ha_gce/data/aws_s3_bucket_object_ha-gce.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content

@@ -200,9 +200,6 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          items:
-          - key: Corefile
-            path: Corefile
           name: coredns
         name: config-volume
 

tests/integration/update_cluster/irsa/data/aws_s3_bucket_object_minimal.example.com-addons-bootstrap_content

@@ -20,7 +20,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.12
     manifest: coredns.addons.k8s.io/k8s-1.12.yaml
-    manifestHash: 88ffe1a3752cf290450cc94bd53aea49a665e411dbf4cfe9c1a2cc5b027f12ef
+    manifestHash: 12b67f439637253329bf6fb2ee23b3ef65959621720c863263c13da8271bff89
     name: coredns.addons.k8s.io
     selector:
       k8s-addon: coredns.addons.k8s.io

tests/integration/update_cluster/irsa/data/aws_s3_bucket_object_minimal.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content

@@ -200,9 +200,6 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          items:
-          - key: Corefile
-            path: Corefile
           name: coredns
         name: config-volume
 

tests/integration/update_cluster/many-addons-ccm-irsa/data/aws_s3_bucket_object_minimal.example.com-addons-bootstrap_content

@@ -20,7 +20,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.12
     manifest: coredns.addons.k8s.io/k8s-1.12.yaml
-    manifestHash: e4297ef11985ee4a7c4e8707453a8d2c16bcb12eccc050adffac3e6ac2fb9a18
+    manifestHash: 262fb03230d0c4a72a9579736b4bc8fdf04e676b8e417790d08c14b4ff03198d
     name: coredns.addons.k8s.io
     selector:
       k8s-addon: coredns.addons.k8s.io

tests/integration/update_cluster/many-addons-ccm-irsa/data/aws_s3_bucket_object_minimal.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content

@@ -200,9 +200,6 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          items:
-          - key: Corefile
-            path: Corefile
           name: coredns
         name: config-volume
 

tests/integration/update_cluster/many-addons-ccm/data/aws_s3_bucket_object_minimal.example.com-addons-bootstrap_content

@@ -20,7 +20,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.12
     manifest: coredns.addons.k8s.io/k8s-1.12.yaml
-    manifestHash: e4297ef11985ee4a7c4e8707453a8d2c16bcb12eccc050adffac3e6ac2fb9a18
+    manifestHash: 262fb03230d0c4a72a9579736b4bc8fdf04e676b8e417790d08c14b4ff03198d
     name: coredns.addons.k8s.io
     selector:
       k8s-addon: coredns.addons.k8s.io

tests/integration/update_cluster/many-addons-ccm/data/aws_s3_bucket_object_minimal.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content

@@ -200,9 +200,6 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          items:
-          - key: Corefile
-            path: Corefile
           name: coredns
         name: config-volume
 

tests/integration/update_cluster/many-addons/data/aws_s3_bucket_object_minimal.example.com-addons-bootstrap_content

@@ -20,7 +20,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.12
     manifest: coredns.addons.k8s.io/k8s-1.12.yaml
-    manifestHash: e4297ef11985ee4a7c4e8707453a8d2c16bcb12eccc050adffac3e6ac2fb9a18
+    manifestHash: 262fb03230d0c4a72a9579736b4bc8fdf04e676b8e417790d08c14b4ff03198d
     name: coredns.addons.k8s.io
     selector:
       k8s-addon: coredns.addons.k8s.io

tests/integration/update_cluster/many-addons/data/aws_s3_bucket_object_minimal.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content

@@ -200,9 +200,6 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          items:
-          - key: Corefile
-            path: Corefile
           name: coredns
         name: config-volume
 

tests/integration/update_cluster/minimal-1.23/data/aws_s3_bucket_object_minimal.example.com-addons-bootstrap_content

@@ -20,7 +20,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.12
     manifest: coredns.addons.k8s.io/k8s-1.12.yaml
-    manifestHash: 88ffe1a3752cf290450cc94bd53aea49a665e411dbf4cfe9c1a2cc5b027f12ef
+    manifestHash: 12b67f439637253329bf6fb2ee23b3ef65959621720c863263c13da8271bff89
     name: coredns.addons.k8s.io
     selector:
       k8s-addon: coredns.addons.k8s.io

tests/integration/update_cluster/minimal-1.23/data/aws_s3_bucket_object_minimal.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content

@@ -200,9 +200,6 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          items:
-          - key: Corefile
-            path: Corefile
           name: coredns
         name: config-volume
 

tests/integration/update_cluster/minimal-gp3/data/aws_s3_bucket_object_minimal.example.com-addons-bootstrap_content

@@ -20,7 +20,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.12
     manifest: coredns.addons.k8s.io/k8s-1.12.yaml
-    manifestHash: 88ffe1a3752cf290450cc94bd53aea49a665e411dbf4cfe9c1a2cc5b027f12ef
+    manifestHash: 12b67f439637253329bf6fb2ee23b3ef65959621720c863263c13da8271bff89
     name: coredns.addons.k8s.io
     selector:
       k8s-addon: coredns.addons.k8s.io

tests/integration/update_cluster/minimal-gp3/data/aws_s3_bucket_object_minimal.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content

@@ -200,9 +200,6 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          items:
-          - key: Corefile
-            path: Corefile
           name: coredns
         name: config-volume
 

tests/integration/update_cluster/minimal-ipv6-calico/data/aws_s3_bucket_object_minimal-ipv6.example.com-addons-bootstrap_content

@@ -20,7 +20,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.12
     manifest: coredns.addons.k8s.io/k8s-1.12.yaml
-    manifestHash: e31327420b42b8d1b813625c65601166c52b054ae9ac95a57048d72e70b7033c
+    manifestHash: 63d09ebb456eb06d694bce805fda8888754e5250eb6a37c120146bbf6f655af4
     name: coredns.addons.k8s.io
     selector:
       k8s-addon: coredns.addons.k8s.io

tests/integration/update_cluster/minimal-ipv6-calico/data/aws_s3_bucket_object_minimal-ipv6.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content

@@ -200,9 +200,6 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          items:
-          - key: Corefile
-            path: Corefile
           name: coredns
         name: config-volume
 

tests/integration/update_cluster/minimal-ipv6-cilium/data/aws_s3_bucket_object_minimal-ipv6.example.com-addons-bootstrap_content

@@ -20,7 +20,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.12
     manifest: coredns.addons.k8s.io/k8s-1.12.yaml
-    manifestHash: e31327420b42b8d1b813625c65601166c52b054ae9ac95a57048d72e70b7033c
+    manifestHash: 63d09ebb456eb06d694bce805fda8888754e5250eb6a37c120146bbf6f655af4
     name: coredns.addons.k8s.io
     selector:
       k8s-addon: coredns.addons.k8s.io

tests/integration/update_cluster/minimal-ipv6-cilium/data/aws_s3_bucket_object_minimal-ipv6.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content

@@ -200,9 +200,6 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          items:
-          - key: Corefile
-            path: Corefile
           name: coredns
         name: config-volume
 

tests/integration/update_cluster/minimal-ipv6-private/data/aws_s3_bucket_object_minimal-ipv6.example.com-addons-bootstrap_content

@@ -20,7 +20,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.12
     manifest: coredns.addons.k8s.io/k8s-1.12.yaml
-    manifestHash: e31327420b42b8d1b813625c65601166c52b054ae9ac95a57048d72e70b7033c
+    manifestHash: 63d09ebb456eb06d694bce805fda8888754e5250eb6a37c120146bbf6f655af4
     name: coredns.addons.k8s.io
     selector:
       k8s-addon: coredns.addons.k8s.io

tests/integration/update_cluster/minimal-ipv6-private/data/aws_s3_bucket_object_minimal-ipv6.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content

@@ -200,9 +200,6 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          items:
-          - key: Corefile
-            path: Corefile
           name: coredns
         name: config-volume
 

tests/integration/update_cluster/minimal-ipv6/data/aws_s3_bucket_object_minimal-ipv6.example.com-addons-bootstrap_content

@@ -20,7 +20,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.12
     manifest: coredns.addons.k8s.io/k8s-1.12.yaml
-    manifestHash: e31327420b42b8d1b813625c65601166c52b054ae9ac95a57048d72e70b7033c
+    manifestHash: 63d09ebb456eb06d694bce805fda8888754e5250eb6a37c120146bbf6f655af4
     name: coredns.addons.k8s.io
     selector:
       k8s-addon: coredns.addons.k8s.io

tests/integration/update_cluster/minimal-ipv6/data/aws_s3_bucket_object_minimal-ipv6.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content

@@ -200,9 +200,6 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          items:
-          - key: Corefile
-            path: Corefile
           name: coredns
         name: config-volume
 

tests/integration/update_cluster/minimal-warmpool/data/aws_s3_bucket_object_minimal-warmpool.example.com-addons-bootstrap_content

@@ -20,7 +20,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.12
     manifest: coredns.addons.k8s.io/k8s-1.12.yaml
-    manifestHash: 88ffe1a3752cf290450cc94bd53aea49a665e411dbf4cfe9c1a2cc5b027f12ef
+    manifestHash: 12b67f439637253329bf6fb2ee23b3ef65959621720c863263c13da8271bff89
     name: coredns.addons.k8s.io
     selector:
       k8s-addon: coredns.addons.k8s.io

tests/integration/update_cluster/minimal-warmpool/data/aws_s3_bucket_object_minimal-warmpool.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content

@@ -200,9 +200,6 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          items:
-          - key: Corefile
-            path: Corefile
           name: coredns
         name: config-volume
 

tests/integration/update_cluster/minimal/data/aws_s3_bucket_object_minimal.example.com-addons-bootstrap_content

@@ -20,7 +20,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.12
     manifest: coredns.addons.k8s.io/k8s-1.12.yaml
-    manifestHash: 88ffe1a3752cf290450cc94bd53aea49a665e411dbf4cfe9c1a2cc5b027f12ef
+    manifestHash: 12b67f439637253329bf6fb2ee23b3ef65959621720c863263c13da8271bff89
     name: coredns.addons.k8s.io
     selector:
       k8s-addon: coredns.addons.k8s.io

tests/integration/update_cluster/minimal/data/aws_s3_bucket_object_minimal.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content

@@ -200,9 +200,6 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          items:
-          - key: Corefile
-            path: Corefile
           name: coredns
         name: config-volume
 

tests/integration/update_cluster/minimal_gce/data/aws_s3_bucket_object_minimal-gce.example.com-addons-bootstrap_content

@@ -20,7 +20,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.12
     manifest: coredns.addons.k8s.io/k8s-1.12.yaml
-    manifestHash: 88ffe1a3752cf290450cc94bd53aea49a665e411dbf4cfe9c1a2cc5b027f12ef
+    manifestHash: 12b67f439637253329bf6fb2ee23b3ef65959621720c863263c13da8271bff89
     name: coredns.addons.k8s.io
     selector:
       k8s-addon: coredns.addons.k8s.io

tests/integration/update_cluster/minimal_gce/data/aws_s3_bucket_object_minimal-gce.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content

@@ -200,9 +200,6 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          items:
-          - key: Corefile
-            path: Corefile
           name: coredns
         name: config-volume
 

tests/integration/update_cluster/minimal_gce_private/data/aws_s3_bucket_object_minimal-gce-private.example.com-addons-bootstrap_content

@@ -20,7 +20,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.12
     manifest: coredns.addons.k8s.io/k8s-1.12.yaml
-    manifestHash: 88ffe1a3752cf290450cc94bd53aea49a665e411dbf4cfe9c1a2cc5b027f12ef
+    manifestHash: 12b67f439637253329bf6fb2ee23b3ef65959621720c863263c13da8271bff89
     name: coredns.addons.k8s.io
     selector:
       k8s-addon: coredns.addons.k8s.io

tests/integration/update_cluster/minimal_gce_private/data/aws_s3_bucket_object_minimal-gce-private.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content

@@ -200,9 +200,6 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          items:
-          - key: Corefile
-            path: Corefile
           name: coredns
         name: config-volume
 

tests/integration/update_cluster/minimal_gossip/data/aws_s3_bucket_object_minimal.k8s.local-addons-bootstrap_content

@@ -6,7 +6,7 @@ spec:
   addons:
   - id: k8s-1.16
     manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml
-    manifestHash: a2b7d85f845283e6a2e41036215e8b27c7808b199e0d613bcf3d1d40dd2f653b
+    manifestHash: 9379d3b1d4508369654ace436cf45137009a50ff1b0e73b88dac88350ebefabd
     name: kops-controller.addons.k8s.io
     needsRollingUpdate: control-plane
     selector:
@@ -20,7 +20,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.12
     manifest: coredns.addons.k8s.io/k8s-1.12.yaml
-    manifestHash: 88ffe1a3752cf290450cc94bd53aea49a665e411dbf4cfe9c1a2cc5b027f12ef
+    manifestHash: 237badcf1d7899f372b94c7f6b3d7dbb0c48bc490d8cbbb9f8d90ed5e1ecb6b3
     name: coredns.addons.k8s.io
     selector:
       k8s-addon: coredns.addons.k8s.io

tests/integration/update_cluster/minimal_gossip/data/aws_s3_bucket_object_minimal.k8s.local-addons-coredns.addons.k8s.io-k8s-1.12_content

@@ -80,6 +80,10 @@ data:
           fallthrough in-addr.arpa ip6.arpa
           ttl 30
         }
+        hosts /etc/coredns/hosts k8s.local {
+          ttl 30
+          fallthrough
+        }
         prometheus :9153
         forward . /etc/resolv.conf {
           max_concurrent 1000
@@ -200,9 +204,6 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          items:
-          - key: Corefile
-            path: Corefile
           name: coredns
         name: config-volume
 

tests/integration/update_cluster/minimal_gossip/data/aws_s3_bucket_object_minimal.k8s.local-addons-kops-controller.addons.k8s.io-k8s-1.16_content

@@ -1,7 +1,7 @@
 apiVersion: v1
 data:
   config.yaml: |
-    {"cloud":"aws","configBase":"memfs://clusters.example.com/minimal.k8s.local","server":{"Listen":":3988","provider":{"aws":{"nodesRoles":["nodes.minimal.k8s.local"],"Region":"us-test-1"}},"serverKeyPath":"/etc/kubernetes/kops-controller/pki/kops-controller.key","serverCertificatePath":"/etc/kubernetes/kops-controller/pki/kops-controller.crt","caBasePath":"/etc/kubernetes/kops-controller/pki","signingCAs":["kubernetes-ca"],"certNames":["kubelet","kubelet-server","kube-proxy"]}}
+    {"cloud":"aws","configBase":"memfs://clusters.example.com/minimal.k8s.local","server":{"Listen":":3988","provider":{"aws":{"nodesRoles":["nodes.minimal.k8s.local"],"Region":"us-test-1"}},"serverKeyPath":"/etc/kubernetes/kops-controller/pki/kops-controller.key","serverCertificatePath":"/etc/kubernetes/kops-controller/pki/kops-controller.crt","caBasePath":"/etc/kubernetes/kops-controller/pki","signingCAs":["kubernetes-ca"],"certNames":["kubelet","kubelet-server","kube-proxy"]},"discovery":{"enabled":true}}
 kind: ConfigMap
 metadata:
   creationTimestamp: null
@@ -119,6 +119,14 @@ rules:
   - list
   - watch
   - patch
+- apiGroups:
+  - ""
+  resources:
+  - endpoints
+  verbs:
+  - get
+  - list
+  - watch
 
 ---
 
@@ -185,6 +193,16 @@ rules:
   - leases
   verbs:
   - create
+- apiGroups:
+  - ""
+  resourceNames:
+  - coredns
+  resources:
+  - configmaps
+  verbs:
+  - get
+  - watch
+  - patch
 
 ---
 
@@ -206,3 +224,51 @@ subjects:
 - apiGroup: rbac.authorization.k8s.io
   kind: User
   name: system:serviceaccount:kube-system:kops-controller
+
+---
+
+apiVersion: v1
+kind: Service
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: kops-controller.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    discovery.kops.k8s.io/internal-name: api.internal.minimal.k8s.local
+    k8s-addon: kops-controller.addons.k8s.io
+  name: api-internal
+  namespace: kube-system
+spec:
+  clusterIP: None
+  ports:
+  - name: https
+    port: 443
+    protocol: TCP
+    targetPort: 443
+  selector:
+    k8s-app: kops-controller
+  type: ClusterIP
+
+---
+
+apiVersion: v1
+kind: Service
+metadata:
+  creationTimestamp: null
+  labels:
+    addon.kops.k8s.io/name: kops-controller.addons.k8s.io
+    app.kubernetes.io/managed-by: kops
+    discovery.kops.k8s.io/internal-name: kops-controller.internal.minimal.k8s.local
+    k8s-addon: kops-controller.addons.k8s.io
+  name: kops-controller-internal
+  namespace: kube-system
+spec:
+  clusterIP: None
+  ports:
+  - name: https
+    port: 3988
+    protocol: TCP
+    targetPort: 3988
+  selector:
+    k8s-app: kops-controller
+  type: ClusterIP

tests/integration/update_cluster/mixed_instances/data/aws_s3_bucket_object_mixedinstances.example.com-addons-bootstrap_content

@@ -20,7 +20,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.12
     manifest: coredns.addons.k8s.io/k8s-1.12.yaml
-    manifestHash: 88ffe1a3752cf290450cc94bd53aea49a665e411dbf4cfe9c1a2cc5b027f12ef
+    manifestHash: 12b67f439637253329bf6fb2ee23b3ef65959621720c863263c13da8271bff89
     name: coredns.addons.k8s.io
     selector:
       k8s-addon: coredns.addons.k8s.io

tests/integration/update_cluster/mixed_instances/data/aws_s3_bucket_object_mixedinstances.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content

@@ -200,9 +200,6 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          items:
-          - key: Corefile
-            path: Corefile
           name: coredns
         name: config-volume
 

tests/integration/update_cluster/mixed_instances_spot/data/aws_s3_bucket_object_mixedinstances.example.com-addons-bootstrap_content

@@ -20,7 +20,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.12
     manifest: coredns.addons.k8s.io/k8s-1.12.yaml
-    manifestHash: 88ffe1a3752cf290450cc94bd53aea49a665e411dbf4cfe9c1a2cc5b027f12ef
+    manifestHash: 12b67f439637253329bf6fb2ee23b3ef65959621720c863263c13da8271bff89
     name: coredns.addons.k8s.io
     selector:
       k8s-addon: coredns.addons.k8s.io

tests/integration/update_cluster/mixed_instances_spot/data/aws_s3_bucket_object_mixedinstances.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content

@@ -200,9 +200,6 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          items:
-          - key: Corefile
-            path: Corefile
           name: coredns
         name: config-volume
 

tests/integration/update_cluster/nth_sqs_resources/data/aws_s3_bucket_object_nthsqsresources.longclustername.example.com-addons-bootstrap_content

@@ -20,7 +20,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.12
     manifest: coredns.addons.k8s.io/k8s-1.12.yaml
-    manifestHash: 88ffe1a3752cf290450cc94bd53aea49a665e411dbf4cfe9c1a2cc5b027f12ef
+    manifestHash: 12b67f439637253329bf6fb2ee23b3ef65959621720c863263c13da8271bff89
     name: coredns.addons.k8s.io
     selector:
       k8s-addon: coredns.addons.k8s.io

tests/integration/update_cluster/nth_sqs_resources/data/aws_s3_bucket_object_nthsqsresources.longclustername.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content

@@ -200,9 +200,6 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          items:
-          - key: Corefile
-            path: Corefile
           name: coredns
         name: config-volume
 

tests/integration/update_cluster/nvidia/data/aws_s3_bucket_object_minimal.example.com-addons-bootstrap_content

@@ -20,7 +20,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.12
     manifest: coredns.addons.k8s.io/k8s-1.12.yaml
-    manifestHash: 88ffe1a3752cf290450cc94bd53aea49a665e411dbf4cfe9c1a2cc5b027f12ef
+    manifestHash: 12b67f439637253329bf6fb2ee23b3ef65959621720c863263c13da8271bff89
     name: coredns.addons.k8s.io
     selector:
       k8s-addon: coredns.addons.k8s.io

tests/integration/update_cluster/nvidia/data/aws_s3_bucket_object_minimal.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content

@@ -200,9 +200,6 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          items:
-          - key: Corefile
-            path: Corefile
           name: coredns
         name: config-volume
 

tests/integration/update_cluster/private-shared-ip/data/aws_s3_bucket_object_private-shared-ip.example.com-addons-bootstrap_content

@@ -20,7 +20,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.12
     manifest: coredns.addons.k8s.io/k8s-1.12.yaml
-    manifestHash: 88ffe1a3752cf290450cc94bd53aea49a665e411dbf4cfe9c1a2cc5b027f12ef
+    manifestHash: 12b67f439637253329bf6fb2ee23b3ef65959621720c863263c13da8271bff89
     name: coredns.addons.k8s.io
     selector:
       k8s-addon: coredns.addons.k8s.io

tests/integration/update_cluster/private-shared-ip/data/aws_s3_bucket_object_private-shared-ip.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content

@@ -200,9 +200,6 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          items:
-          - key: Corefile
-            path: Corefile
           name: coredns
         name: config-volume
 

tests/integration/update_cluster/private-shared-subnet/data/aws_s3_bucket_object_private-shared-subnet.example.com-addons-bootstrap_content

@@ -20,7 +20,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.12
     manifest: coredns.addons.k8s.io/k8s-1.12.yaml
-    manifestHash: 88ffe1a3752cf290450cc94bd53aea49a665e411dbf4cfe9c1a2cc5b027f12ef
+    manifestHash: 12b67f439637253329bf6fb2ee23b3ef65959621720c863263c13da8271bff89
     name: coredns.addons.k8s.io
     selector:
       k8s-addon: coredns.addons.k8s.io

tests/integration/update_cluster/private-shared-subnet/data/aws_s3_bucket_object_private-shared-subnet.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content

@@ -200,9 +200,6 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          items:
-          - key: Corefile
-            path: Corefile
           name: coredns
         name: config-volume
 

tests/integration/update_cluster/privatecalico/data/aws_s3_bucket_object_privatecalico.example.com-addons-bootstrap_content

@@ -20,7 +20,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.12
     manifest: coredns.addons.k8s.io/k8s-1.12.yaml
-    manifestHash: 88ffe1a3752cf290450cc94bd53aea49a665e411dbf4cfe9c1a2cc5b027f12ef
+    manifestHash: 12b67f439637253329bf6fb2ee23b3ef65959621720c863263c13da8271bff89
     name: coredns.addons.k8s.io
     selector:
       k8s-addon: coredns.addons.k8s.io

tests/integration/update_cluster/privatecalico/data/aws_s3_bucket_object_privatecalico.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content

@@ -200,9 +200,6 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          items:
-          - key: Corefile
-            path: Corefile
           name: coredns
         name: config-volume
 

tests/integration/update_cluster/privatecanal/data/aws_s3_bucket_object_privatecanal.example.com-addons-bootstrap_content

@@ -20,7 +20,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.12
     manifest: coredns.addons.k8s.io/k8s-1.12.yaml
-    manifestHash: 88ffe1a3752cf290450cc94bd53aea49a665e411dbf4cfe9c1a2cc5b027f12ef
+    manifestHash: 12b67f439637253329bf6fb2ee23b3ef65959621720c863263c13da8271bff89
     name: coredns.addons.k8s.io
     selector:
       k8s-addon: coredns.addons.k8s.io

tests/integration/update_cluster/privatecanal/data/aws_s3_bucket_object_privatecanal.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content

@@ -200,9 +200,6 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          items:
-          - key: Corefile
-            path: Corefile
           name: coredns
         name: config-volume
 

tests/integration/update_cluster/privatecilium/data/aws_s3_bucket_object_privatecilium.example.com-addons-bootstrap_content

@@ -20,7 +20,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.12
     manifest: coredns.addons.k8s.io/k8s-1.12.yaml
-    manifestHash: 88ffe1a3752cf290450cc94bd53aea49a665e411dbf4cfe9c1a2cc5b027f12ef
+    manifestHash: 12b67f439637253329bf6fb2ee23b3ef65959621720c863263c13da8271bff89
     name: coredns.addons.k8s.io
     selector:
       k8s-addon: coredns.addons.k8s.io

tests/integration/update_cluster/privatecilium/data/aws_s3_bucket_object_privatecilium.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content

@@ -200,9 +200,6 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          items:
-          - key: Corefile
-            path: Corefile
           name: coredns
         name: config-volume
 

tests/integration/update_cluster/privateciliumadvanced/data/aws_s3_bucket_object_privateciliumadvanced.example.com-addons-bootstrap_content

@@ -20,7 +20,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.12
     manifest: coredns.addons.k8s.io/k8s-1.12.yaml
-    manifestHash: 88ffe1a3752cf290450cc94bd53aea49a665e411dbf4cfe9c1a2cc5b027f12ef
+    manifestHash: 12b67f439637253329bf6fb2ee23b3ef65959621720c863263c13da8271bff89
     name: coredns.addons.k8s.io
     selector:
       k8s-addon: coredns.addons.k8s.io

tests/integration/update_cluster/privateciliumadvanced/data/aws_s3_bucket_object_privateciliumadvanced.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content

@@ -200,9 +200,6 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          items:
-          - key: Corefile
-            path: Corefile
           name: coredns
         name: config-volume
 

tests/integration/update_cluster/privatedns1/data/aws_s3_bucket_object_privatedns1.example.com-addons-bootstrap_content

@@ -20,7 +20,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.12
     manifest: coredns.addons.k8s.io/k8s-1.12.yaml
-    manifestHash: 88ffe1a3752cf290450cc94bd53aea49a665e411dbf4cfe9c1a2cc5b027f12ef
+    manifestHash: 12b67f439637253329bf6fb2ee23b3ef65959621720c863263c13da8271bff89
     name: coredns.addons.k8s.io
     selector:
       k8s-addon: coredns.addons.k8s.io

tests/integration/update_cluster/privatedns1/data/aws_s3_bucket_object_privatedns1.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content

@@ -200,9 +200,6 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          items:
-          - key: Corefile
-            path: Corefile
           name: coredns
         name: config-volume
 

tests/integration/update_cluster/privatedns2/data/aws_s3_bucket_object_privatedns2.example.com-addons-bootstrap_content

@@ -20,7 +20,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.12
     manifest: coredns.addons.k8s.io/k8s-1.12.yaml
-    manifestHash: 88ffe1a3752cf290450cc94bd53aea49a665e411dbf4cfe9c1a2cc5b027f12ef
+    manifestHash: 12b67f439637253329bf6fb2ee23b3ef65959621720c863263c13da8271bff89
     name: coredns.addons.k8s.io
     selector:
       k8s-addon: coredns.addons.k8s.io

tests/integration/update_cluster/privatedns2/data/aws_s3_bucket_object_privatedns2.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content

@@ -200,9 +200,6 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          items:
-          - key: Corefile
-            path: Corefile
           name: coredns
         name: config-volume
 

tests/integration/update_cluster/privateflannel/data/aws_s3_bucket_object_privateflannel.example.com-addons-bootstrap_content

@@ -20,7 +20,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.12
     manifest: coredns.addons.k8s.io/k8s-1.12.yaml
-    manifestHash: 88ffe1a3752cf290450cc94bd53aea49a665e411dbf4cfe9c1a2cc5b027f12ef
+    manifestHash: 12b67f439637253329bf6fb2ee23b3ef65959621720c863263c13da8271bff89
     name: coredns.addons.k8s.io
     selector:
       k8s-addon: coredns.addons.k8s.io

tests/integration/update_cluster/privateflannel/data/aws_s3_bucket_object_privateflannel.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content

@@ -200,9 +200,6 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          items:
-          - key: Corefile
-            path: Corefile
           name: coredns
         name: config-volume
 

tests/integration/update_cluster/privatekopeio/data/aws_s3_bucket_object_privatekopeio.example.com-addons-bootstrap_content

@@ -20,7 +20,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.12
     manifest: coredns.addons.k8s.io/k8s-1.12.yaml
-    manifestHash: 88ffe1a3752cf290450cc94bd53aea49a665e411dbf4cfe9c1a2cc5b027f12ef
+    manifestHash: 12b67f439637253329bf6fb2ee23b3ef65959621720c863263c13da8271bff89
     name: coredns.addons.k8s.io
     selector:
       k8s-addon: coredns.addons.k8s.io

tests/integration/update_cluster/privatekopeio/data/aws_s3_bucket_object_privatekopeio.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content

@@ -200,9 +200,6 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          items:
-          - key: Corefile
-            path: Corefile
           name: coredns
         name: config-volume
 

tests/integration/update_cluster/privateweave/data/aws_s3_bucket_object_privateweave.example.com-addons-bootstrap_content

@@ -20,7 +20,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.12
     manifest: coredns.addons.k8s.io/k8s-1.12.yaml
-    manifestHash: 88ffe1a3752cf290450cc94bd53aea49a665e411dbf4cfe9c1a2cc5b027f12ef
+    manifestHash: 12b67f439637253329bf6fb2ee23b3ef65959621720c863263c13da8271bff89
     name: coredns.addons.k8s.io
     selector:
       k8s-addon: coredns.addons.k8s.io

tests/integration/update_cluster/privateweave/data/aws_s3_bucket_object_privateweave.example.com-addons-coredns.addons.k8s.io-k8s-1.12_content

@@ -200,9 +200,6 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          items:
-          - key: Corefile
-            path: Corefile
           name: coredns
         name: config-volume
 

tests/integration/update_cluster/shared_subnet/data/aws_s3_bucket_object_sharedsubnet.example.com-addons-bootstrap_content

@@ -20,7 +20,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.12
     manifest: coredns.addons.k8s.io/k8s-1.12.yaml
-    manifestHash: 88ffe1a3752cf290450cc94bd53aea49a665e411dbf4cfe9c1a2cc5b027f12ef
+    manifestHash: 12b67f439637253329bf6fb2ee23b3ef65959621720c863263c13da8271bff89
     name: coredns.addons.k8s.io
     selector:
       k8s-addon: coredns.addons.k8s.io