gossip: Seed /etc/hosts in nodeup

In some scenarios (e.g. cilium), we rely on the internal DNS name
being available, but this isn't the case with gossip clusters.

nodeup can seed /etc/hosts for the control-plane nodes, breaking the
deadlock.

nodeup/pkg/model/dns/BUILD.bazel

@@ -0,0 +1,14 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["builder.go"],
+    importpath = "k8s.io/kops/nodeup/pkg/model/dns",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//nodeup/pkg/model:go_default_library",
+        "//pkg/dns:go_default_library",
+        "//upup/pkg/fi:go_default_library",
+        "//upup/pkg/fi/nodeup/nodetasks/dnstasks:go_default_library",
+    ],
+)

nodeup/pkg/model/dns/builder.go

@@ -0,0 +1,64 @@
+/*
+Copyright 2021 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dns
+
+import (
+	"k8s.io/kops/nodeup/pkg/model"
+	"k8s.io/kops/pkg/dns"
+	"k8s.io/kops/upup/pkg/fi"
+	"k8s.io/kops/upup/pkg/fi/nodeup/nodetasks/dnstasks"
+)
+
+// GossipBuilder seeds some hostnames into /etc/hosts, avoiding some circular dependencies.
+type GossipBuilder struct {
+	*model.NodeupModelContext
+}
+
+var _ fi.ModelBuilder = &GossipBuilder{}
+
+// Build is responsible for configuring the gossip DNS tasks.
+func (b *GossipBuilder) Build(c *fi.ModelBuilderContext) error {
+	useGossip := dns.IsGossipHostname(b.Cluster.Spec.MasterInternalName)
+	if !useGossip {
+		return nil
+	}
+
+	if b.IsMaster {
+		task := &dnstasks.UpdateEtcHostsTask{
+			Name: "control-plane-bootstrap",
+		}
+
+		if b.Cluster.Spec.MasterInternalName != "" {
+			task.Records = append(task.Records, dnstasks.HostRecord{
+				Hostname:  b.Cluster.Spec.MasterInternalName,
+				Addresses: []string{"127.0.0.1"},
+			})
+		}
+		if b.Cluster.Spec.MasterPublicName != "" {
+			task.Records = append(task.Records, dnstasks.HostRecord{
+				Hostname:  b.Cluster.Spec.MasterPublicName,
+				Addresses: []string{"127.0.0.1"},
+			})
+		}
+
+		if len(task.Records) != 0 {
+			c.AddTask(task)
+		}
+	}
+
+	return nil
+}

protokube/pkg/gossip/dns/hosts.go

@@ -31,23 +31,29 @@ var _ DNSTarget = &HostsFile{}
 func (h *HostsFile) Update(snapshot *DNSViewSnapshot) error {
 	klog.V(2).Infof("Updating hosts file with snapshot version %v", snapshot.version)
 
-	addrToHosts := make(map[string][]string)
+	mutator := func(existing []string) (*hosts.HostMap, error) {
+		hostMap := &hosts.HostMap{}
+		badLines := hostMap.Parse(existing)
+		if len(badLines) != 0 {
+			klog.Warningf("ignoring unexpected lines in /etc/hosts: %v", badLines)
+		}
 
-	zones := snapshot.ListZones()
-	for _, zone := range zones {
-		records := snapshot.RecordsForZone(zone)
+		zones := snapshot.ListZones()
+		for _, zone := range zones {
+			records := snapshot.RecordsForZone(zone)
 
-		for _, record := range records {
-			if record.RrsType != "A" {
-				klog.Warningf("skipping record of unhandled type: %v", record)
-				continue
-			}
+			for _, record := range records {
+				if record.RrsType != "A" {
+					klog.Warningf("skipping record of unhandled type: %v", record)
+					continue
+				}
 
-			for _, addr := range record.Rrdatas {
-				addrToHosts[addr] = append(addrToHosts[addr], record.Name)
+				hostMap.ReplaceRecords(record.Name, record.Rrdatas)
 			}
 		}
+
+		return hostMap, nil
 	}
 
-	return hosts.UpdateHostsFileWithRecords(h.Path, addrToHosts)
+	return hosts.UpdateHostsFileWithRecords(h.Path, mutator)
 }

protokube/pkg/gossip/dns/hosts/hosts.go

@@ -37,7 +37,77 @@ const (
 
 var hostsFileMutex sync.Mutex
 
-func UpdateHostsFileWithRecords(p string, addrToHosts map[string][]string) error {
+// HostMap holds a set of host to address mappings, a simplification of /etc/hosts.
+type HostMap struct {
+	records []hostMapRecord
+}
+
+// ParseHostMap parses lines from /etc/hosts (expected to be our guarded block) into a HostMap structure.
+// It parses as much as it can, and returns invalid lines (which should ideally be empty)
+func (m *HostMap) Parse(existing []string) []string {
+	var badLines []string
+
+	for _, line := range existing {
+		tokens := strings.Fields(line)
+		if len(tokens) == 0 {
+			continue
+		}
+		if strings.HasPrefix(tokens[0], "#") {
+			// Comments shouldn't really happen in our guarded block
+			if line == GUARD_BEGIN || line == GUARD_END {
+				klog.Warningf("ignoring extra guard line in /etc/hosts: %q", line)
+			} else {
+				badLines = append(badLines, line)
+			}
+			continue
+		}
+
+		if len(tokens) == 1 {
+			badLines = append(badLines, line)
+			continue
+		}
+
+		address := tokens[0]
+		for _, hostname := range tokens[1:] {
+			m.records = append(m.records, hostMapRecord{
+				Address:  address,
+				Hostname: hostname,
+			})
+		}
+	}
+
+	return badLines
+}
+
+// hostMap holds a single host-name to address mapping.
+type hostMapRecord struct {
+	Hostname string
+	Address  string
+}
+
+// ReplaceRecords replaces all the addresses for the given hostname.
+func (m *HostMap) ReplaceRecords(hostname string, addresses []string) {
+	var newRecords []hostMapRecord
+
+	for _, address := range addresses {
+		newRecords = append(newRecords, hostMapRecord{
+			Hostname: hostname,
+			Address:  address,
+		})
+	}
+
+	for _, record := range m.records {
+		if record.Hostname == hostname {
+			continue
+		}
+		newRecords = append(newRecords, record)
+	}
+
+	m.records = newRecords
+}
+
+// UpdateHostsFileWithRecords updates /etc/hosts by applying the given mutation function.
+func UpdateHostsFileWithRecords(p string, mutator func(guarded []string) (*HostMap, error)) error {
 	// For safety / sanity, we avoid concurrent updates from one process
 	hostsFileMutex.Lock()
 	defer hostsFileMutex.Unlock()
@@ -52,6 +122,7 @@ func UpdateHostsFileWithRecords(p string, addrToHosts map[string][]string) error
 		return fmt.Errorf("error reading file %q: %v", p, err)
 	}
 
+	var guarded []string
 	var out []string
 	inGuardBlock := false
 	for _, line := range strings.Split(string(data), "\n") {
@@ -63,7 +134,9 @@ func UpdateHostsFileWithRecords(p string, addrToHosts map[string][]string) error
 			inGuardBlock = true
 		}
 
-		if !inGuardBlock {
+		if inGuardBlock {
+			guarded = append(guarded, line)
+		} else {
 			out = append(out, line)
 		}
 
@@ -92,7 +165,16 @@ func UpdateHostsFileWithRecords(p string, addrToHosts map[string][]string) error
 	}
 	out = append(out, "")
 
+	hosts, err := mutator(guarded)
+	if err != nil {
+		return err
+	}
+
 	var block []string
+	addrToHosts := make(map[string][]string)
+	for _, record := range hosts.records {
+		addrToHosts[record.Address] = append(addrToHosts[record.Address], record.Hostname)
+	}
 	for addr, hosts := range addrToHosts {
 		sort.Strings(hosts)
 		block = append(block, addr+"\t"+strings.Join(hosts, " "))

protokube/pkg/gossip/dns/hosts/hosts_test.go

@@ -20,7 +20,6 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
-	"strings"
 	"testing"
 
 	"k8s.io/kops/pkg/diff"
@@ -28,7 +27,7 @@ import (
 
 func TestRemovesDuplicateGuardedBlocks(t *testing.T) {
 	in := `
-foo 10.2.3.4
+10.2.3.4 foo
 
 # Begin host entries managed by etcd-manager[etcd] - do not edit
 # End host entries managed by etcd-manager[etcd]
@@ -53,7 +52,7 @@ foo 10.2.3.4
 `
 
 	expected := `
-foo 10.2.3.4
+10.2.3.4 foo
 
 # Begin host entries managed by etcd-manager[etcd] - do not edit
 # End host entries managed by etcd-manager[etcd]
@@ -61,9 +60,9 @@ foo 10.2.3.4
 # End host entries managed by etcd-manager[etcd]
 
 # Begin host entries managed by kops - do not edit
-a\t10.0.1.1 10.0.1.2
-b\t10.0.2.1
-c\t
+10.0.1.1	a
+10.0.1.2	a
+10.0.2.1	b
 # End host entries managed by kops
 `
 
@@ -72,7 +71,7 @@ c\t
 
 func TestRecoversFromBadNesting(t *testing.T) {
 	in := `
-foo 10.2.3.4
+10.2.3.4 foo
 
 # End host entries managed by kops
 # Begin host entries managed by kops - do not edit
@@ -94,19 +93,19 @@ foo 10.2.3.4
 # Begin host entries managed by kops - do not edit
 # End host entries managed by kops
 
-bar 10.1.2.3
+10.1.2.3 bar
 `
 
 	expected := `
-foo 10.2.3.4
+10.2.3.4 foo
 
 
-bar 10.1.2.3
+10.1.2.3 bar
 
 # Begin host entries managed by kops - do not edit
-a\t10.0.1.1 10.0.1.2
-b\t10.0.2.1
-c\t
+10.0.1.1	a
+10.0.1.2	a
+10.0.2.1	b
 # End host entries managed by kops
 `
 
@@ -114,8 +113,6 @@ c\t
 }
 
 func runTest(t *testing.T, in string, expected string) {
-	expected = strings.Replace(expected, "\\t", "\t", -1)
-
 	dir, err := ioutil.TempDir("", "")
 	if err != nil {
 		t.Fatalf("error creating temp dir: %v", err)
@@ -128,7 +125,7 @@ func runTest(t *testing.T, in string, expected string) {
 	}()
 
 	p := filepath.Join(dir, "hosts")
-	addrToHosts := map[string][]string{
+	namesToAddresses := map[string][]string{
 		"a": {"10.0.1.2", "10.0.1.1"},
 		"b": {"10.0.2.1"},
 		"c": {},
@@ -140,7 +137,20 @@ func runTest(t *testing.T, in string, expected string) {
 
 	// We run it repeatedly to make sure we don't change it accidentally
 	for i := 0; i < 100; i++ {
-		if err := UpdateHostsFileWithRecords(p, addrToHosts); err != nil {
+		mutator := func(existing []string) (*HostMap, error) {
+			hostMap := &HostMap{}
+			badLines := hostMap.Parse(existing)
+			if len(badLines) != 0 {
+				t.Errorf("unexpected lines in /etc/hosts: %v", badLines)
+			}
+
+			for name, addresses := range namesToAddresses {
+				hostMap.ReplaceRecords(name, addresses)
+			}
+
+			return hostMap, nil
+		}
+		if err := UpdateHostsFileWithRecords(p, mutator); err != nil {
 			t.Fatalf("error updating hosts file: %v", err)
 		}
 

upup/pkg/fi/nodeup/BUILD.bazel

@@ -10,6 +10,7 @@ go_library(
     visibility = ["//visibility:public"],
     deps = [
         "//nodeup/pkg/model:go_default_library",
+        "//nodeup/pkg/model/dns:go_default_library",
         "//nodeup/pkg/model/networking:go_default_library",
         "//pkg/apis/kops:go_default_library",
         "//pkg/apis/kops/registry:go_default_library",

upup/pkg/fi/nodeup/command.go

@@ -34,6 +34,7 @@ import (
 
 	"github.com/aws/aws-sdk-go/service/kms"
 	"k8s.io/kops/nodeup/pkg/model"
+	"k8s.io/kops/nodeup/pkg/model/dns"
 	"k8s.io/kops/nodeup/pkg/model/networking"
 	api "k8s.io/kops/pkg/apis/kops"
 	"k8s.io/kops/pkg/apis/kops/registry"
@@ -300,6 +301,7 @@ func (c *NodeUpCommand) Run(out io.Writer) error {
 	}
 
 	loader := &Loader{}
+	loader.Builders = append(loader.Builders, &dns.GossipBuilder{NodeupModelContext: modelContext})
 	loader.Builders = append(loader.Builders, &model.NTPBuilder{NodeupModelContext: modelContext})
 	loader.Builders = append(loader.Builders, &model.MiscUtilsBuilder{NodeupModelContext: modelContext})
 	loader.Builders = append(loader.Builders, &model.DirectoryBuilder{NodeupModelContext: modelContext})

upup/pkg/fi/nodeup/nodetasks/BUILD.bazel

@@ -35,6 +35,7 @@ go_library(
         "//upup/pkg/fi/cloudup/awsup:go_default_library",
         "//upup/pkg/fi/nodeup/cloudinit:go_default_library",
         "//upup/pkg/fi/nodeup/local:go_default_library",
+        "//upup/pkg/fi/nodeup/nodetasks/dnstasks:go_default_library",
         "//upup/pkg/fi/utils:go_default_library",
         "//util/pkg/distributions:go_default_library",
         "//util/pkg/hashing:go_default_library",

upup/pkg/fi/nodeup/nodetasks/dnstasks/BUILD.bazel

@@ -0,0 +1,15 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["update_etc_hosts_task.go"],
+    importpath = "k8s.io/kops/upup/pkg/fi/nodeup/nodetasks/dnstasks",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//protokube/pkg/gossip/dns/hosts:go_default_library",
+        "//upup/pkg/fi:go_default_library",
+        "//upup/pkg/fi/nodeup/cloudinit:go_default_library",
+        "//upup/pkg/fi/nodeup/local:go_default_library",
+        "//vendor/k8s.io/klog/v2:go_default_library",
+    ],
+)

upup/pkg/fi/nodeup/nodetasks/dnstasks/update_etc_hosts_task.go

@@ -0,0 +1,100 @@
+/*
+Copyright 2021 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dnstasks
+
+import (
+	"fmt"
+
+	"k8s.io/klog/v2"
+	"k8s.io/kops/protokube/pkg/gossip/dns/hosts"
+	"k8s.io/kops/upup/pkg/fi"
+	"k8s.io/kops/upup/pkg/fi/nodeup/cloudinit"
+	"k8s.io/kops/upup/pkg/fi/nodeup/local"
+)
+
+// UpdateEtcHostsTask is responsible for updating /etc/hosts to set some DNS records, for gossip.
+type UpdateEtcHostsTask struct {
+	// Name is a reference for our task
+	Name string
+
+	// Records holds the records that should be updated
+	Records []HostRecord
+}
+
+// HostRecord holds an individual host's addresses.
+type HostRecord struct {
+	fi.NotADependency
+
+	// Hostname is the "DNS" name that we want to configure.
+	Hostname string
+	// Addresses holds the IP addresses to write.
+	// Other IP addresses for the same Name will be removed.
+	Addresses []string
+}
+
+var _ fi.Task = &UpdateEtcHostsTask{}
+
+func (e *UpdateEtcHostsTask) String() string {
+	return fmt.Sprintf("UpdateEtcHostsTask: %s", e.Name)
+}
+
+var _ fi.HasName = &UpdateEtcHostsTask{}
+
+func (f *UpdateEtcHostsTask) GetName() *string {
+	return &f.Name
+}
+
+func (e *UpdateEtcHostsTask) Find(c *fi.Context) (*UpdateEtcHostsTask, error) {
+	// UpdateHostsFileWithRecords skips the update /etc/hosts if there are no changes,
+	// so we don't check existing values here.
+	return nil, nil
+}
+
+func (e *UpdateEtcHostsTask) Run(c *fi.Context) error {
+	return fi.DefaultDeltaRunMethod(e, c)
+}
+
+func (_ *UpdateEtcHostsTask) CheckChanges(a, e, changes *UpdateEtcHostsTask) error {
+	return nil
+}
+
+func (_ *UpdateEtcHostsTask) RenderLocal(t *local.LocalTarget, a, e, changes *UpdateEtcHostsTask) error {
+	etcHostsPath := "/etc/hosts"
+
+	mutator := func(existing []string) (*hosts.HostMap, error) {
+		hostMap := &hosts.HostMap{}
+		badLines := hostMap.Parse(existing)
+		if len(badLines) != 0 {
+			klog.Warningf("ignoring unexpected lines in /etc/hosts: %v", badLines)
+		}
+
+		for _, record := range e.Records {
+			hostMap.ReplaceRecords(record.Hostname, record.Addresses)
+		}
+
+		return hostMap, nil
+	}
+
+	if err := hosts.UpdateHostsFileWithRecords(etcHostsPath, mutator); err != nil {
+		return fmt.Errorf("failed to update /etc/hosts: %w", err)
+	}
+	return nil
+}
+
+func (_ *UpdateEtcHostsTask) RenderCloudInit(t *cloudinit.CloudInitTarget, a, e, changes *UpdateEtcHostsTask) error {
+	return fmt.Errorf("UpdateEtcHostsTask::RenderCloudInit not supported")
+}

upup/pkg/fi/nodeup/nodetasks/service.go

@@ -29,6 +29,7 @@ import (
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kops/upup/pkg/fi/nodeup/cloudinit"
 	"k8s.io/kops/upup/pkg/fi/nodeup/local"
+	"k8s.io/kops/upup/pkg/fi/nodeup/nodetasks/dnstasks"
 	"k8s.io/kops/util/pkg/distributions"
 )
 
@@ -73,7 +74,7 @@ func (p *Service) GetDependencies(tasks map[string]fi.Task) []fi.Task {
 		// launching a custom Kubernetes build), they all depend on
 		// the "docker.service" Service task.
 		switch v := v.(type) {
-		case *Package, *UpdatePackages, *UserTask, *GroupTask, *Chattr, *BindMount, *Archive, *Prefix:
+		case *Package, *UpdatePackages, *UserTask, *GroupTask, *Chattr, *BindMount, *Archive, *Prefix, *dnstasks.UpdateEtcHostsTask:
 			deps = append(deps, v)
 		case *Service, *LoadImageTask, *PullImageTask, *IssueCert, *BootstrapClientTask, *KubeConfig:
 			// ignore