Disable kubelet from starting until after volume mounts

* Change protokube to do `systemctl start kubelet` every sync round
** .. which takes a change to the systemd unit for protokube to mount in D-Bus
* Don't start kubelet in nodeup

Makefile

@@ -30,7 +30,7 @@ MAKEDIR:=$(strip $(shell dirname "$(realpath $(lastword $(MAKEFILE_LIST)))"))
 
 # Keep in sync with upup/models/cloudup/resources/addons/dns-controller/
 DNS_CONTROLLER_TAG=1.4.1
-PROTOKUBE_TAG=1.4.0
+PROTOKUBE_TAG=1.4.1
 
 ifndef VERSION
   VERSION := git-$(shell git describe --always)

protokube/pkg/protokube/kube_boot.go

@@ -17,9 +17,12 @@ limitations under the License.
 package protokube
 
 import (
-	"github.com/golang/glog"
+	"fmt"
 	"net"
+	"os/exec"
 	"time"
+
+	"github.com/golang/glog"
 )
 
 type KubeBoot struct {
@@ -108,18 +111,34 @@ func (k *KubeBoot) syncOnce() error {
 	}
 
 	if k.Master {
-		err := ApplyMasterTaints(k.Kubernetes)
-		if err != nil {
+		if err := ApplyMasterTaints(k.Kubernetes); err != nil {
 			glog.Warningf("error updating master taints: %v", err)
 		}
 	}
 
+	// Ensure kubelet is running. We avoid doing this automatically so
+	// that when kubelet comes up the first time, all volume mounts
+	// and DNS are available, avoiding the scenario where
+	// etcd/apiserver retry too many times and go into backoff.
+	if err := enableKubelet(); err != nil {
+		glog.Warningf("error ensuring kubelet started: %v", err)
+	}
+
 	for _, channel := range k.Channels {
-		err := ApplyChannel(channel)
-		if err != nil {
+		if err := ApplyChannel(channel); err != nil {
 			glog.Warningf("error applying channel %q: %v", channel, err)
 		}
 	}
 
 	return nil
 }
+
+// enableKubelet: Make sure kubelet is running.
+func enableKubelet() error {
+	cmd := exec.Command("systemctl", "start", "--no-block", "kubelet")
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("error starting kubelet: %v\nOutput: %s", err, output)
+	}
+	return nil
+}

upup/models/nodeup/_protokube/services/protokube.service.template

@@ -6,7 +6,7 @@ After=docker.service
 [Service]
 EnvironmentFile=/etc/sysconfig/protokube
 ExecStartPre=/usr/bin/docker pull {{ ProtokubeImage }}
-ExecStart=/usr/bin/docker run -v /:/rootfs/ --net=host --privileged {{ ProtokubeImage }} /usr/bin/protokube "$DAEMON_ARGS"
+ExecStart=/usr/bin/docker run -v /:/rootfs/ -v /var/run/dbus:/var/run/dbus -v /run/systemd:/run/systemd --net=host --privileged {{ ProtokubeImage }} /usr/bin/protokube "$DAEMON_ARGS"
 Restart=always
 RestartSec=2s
 StartLimitInterval=0

upup/models/nodeup/kubelet/services/kubelet.service

@@ -10,6 +10,3 @@ Restart=always
 RestartSec=2s
 StartLimitInterval=0
 KillMode=process
-
-[Install]
-WantedBy=multi-user.target

upup/models/nodeup/kubelet/services/kubelet.service.meta

@@ -0,0 +1,3 @@
+{
+  "running": false
+}

upup/pkg/fi/nodeup/nodetasks/service.go

@@ -45,7 +45,7 @@ const (
 type Service struct {
 	Name       string
 	Definition *string
-	Running    *bool
+	Running    *bool `json:"running"`
 
 	// Enabled configures the service to start at boot (or not start at boot)
 	Enabled *bool

upup/pkg/fi/nodeup/template_functions.go

@@ -19,18 +19,19 @@ package nodeup
 import (
 	"encoding/base64"
 	"fmt"
+	"text/template"
+
 	"github.com/golang/glog"
 	api "k8s.io/kops/pkg/apis/kops"
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kops/upup/pkg/fi/secrets"
 	"k8s.io/kops/util/pkg/vfs"
 	"k8s.io/kubernetes/pkg/util/sets"
-	"text/template"
 )
 
 const TagMaster = "_kubernetes_master"
 
-const DefaultProtokubeImage = "kope/protokube:1.4"
+const DefaultProtokubeImage = "b.gcr.io/kops-images/protokube:1.4.1"
 
 // templateFunctions is a simple helper-class for the functions accessible to templates
 type templateFunctions struct {