Create helper function for critical pod annotations

In particularly I think we want a toleration also; easiest to put the
code in one function.

nodeup/pkg/model/kube_apiserver.go

@@ -31,6 +31,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/client-go/pkg/api/v1"
+	"k8s.io/kops/pkg/kubemanifest"
 )
 
 const PathAuthnConfig = "/etc/kubernetes/authn.config"
@@ -302,6 +303,8 @@ func (b *KubeAPIServerBuilder) buildPod() (*v1.Pod, error) {
 
 	pod.Spec.Containers = append(pod.Spec.Containers, *container)
 
+	kubemanifest.MarkPodAsCritical(pod)
+
 	return pod, nil
 }
 
@@ -311,7 +314,6 @@ func (b *KubeAPIServerBuilder) buildAnnotations() map[string]string {
 	if b.Cluster.Spec.API != nil && b.Cluster.Spec.API.DNS != nil {
 		annotations["dns.alpha.kubernetes.io/external"] = b.Cluster.Spec.MasterPublicName
 	}
-	annotations["scheduler.alpha.kubernetes.io/critical-pod"] = ""
 
 	return annotations
 }

nodeup/pkg/model/kube_controller_manager.go

@@ -29,6 +29,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/client-go/pkg/api/v1"
+	"k8s.io/kops/pkg/kubemanifest"
 )
 
 // KubeControllerManagerBuilder install kube-controller-manager (just the manifest at the moment)
@@ -149,9 +150,6 @@ func (b *KubeControllerManagerBuilder) buildPod() (*v1.Pod, error) {
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      "kube-controller-manager",
 			Namespace: "kube-system",
-			Annotations: map[string]string{
-				"scheduler.alpha.kubernetes.io/critical-pod": "",
-			},
 			Labels: map[string]string{
 				"k8s-app": "kube-controller-manager",
 			},
@@ -207,5 +205,7 @@ func (b *KubeControllerManagerBuilder) buildPod() (*v1.Pod, error) {
 
 	pod.Spec.Containers = append(pod.Spec.Containers, *container)
 
+	kubemanifest.MarkPodAsCritical(pod)
+
 	return pod, nil
 }

nodeup/pkg/model/kube_proxy.go

@@ -29,6 +29,7 @@ import (
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/pkg/api/v1"
+	"k8s.io/kops/pkg/kubemanifest"
 )
 
 // KubeProxyBuilder installs kube-proxy
@@ -158,14 +159,6 @@ func (b *KubeProxyBuilder) buildPod() (*v1.Pod, error) {
 				"k8s-app": "kube-proxy",
 				"tier":    "node",
 			},
-			Annotations: map[string]string{
-				// This annotation ensures that kube-proxy does not get evicted if the node
-				// supports critical pod annotation based priority scheme.
-				// Note that kube-proxy runs as a static pod so this annotation does NOT have
-				// any effect on rescheduler (default scheduler and rescheduler are not
-				// involved in scheduling kube-proxy).
-				"scheduler.alpha.kubernetes.io/critical-pod": "",
-			},
 		},
 		Spec: v1.PodSpec{
 			HostNetwork: true,
@@ -210,6 +203,13 @@ func (b *KubeProxyBuilder) buildPod() (*v1.Pod, error) {
 	//	},
 	//}
 
+	// This annotation ensures that kube-proxy does not get evicted if the node
+	// supports critical pod annotation based priority scheme.
+	// Note that kube-proxy runs as a static pod so this annotation does NOT have
+	// any effect on rescheduler (default scheduler and rescheduler are not
+	// involved in scheduling kube-proxy).
+	kubemanifest.MarkPodAsCritical(pod)
+
 	return pod, nil
 }
 

nodeup/pkg/model/kube_scheduler.go

@@ -28,6 +28,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/client-go/pkg/api/v1"
+	"k8s.io/kops/pkg/kubemanifest"
 )
 
 // KubeSchedulerBuilder install kube-scheduler
@@ -110,9 +111,6 @@ func (b *KubeSchedulerBuilder) buildPod() (*v1.Pod, error) {
 			Labels: map[string]string{
 				"k8s-app": "kube-scheduler",
 			},
-			Annotations: map[string]string{
-				"scheduler.alpha.kubernetes.io/critical-pod": "",
-			},
 		},
 		Spec: v1.PodSpec{
 			HostNetwork: true,
@@ -149,5 +147,7 @@ func (b *KubeSchedulerBuilder) buildPod() (*v1.Pod, error) {
 
 	pod.Spec.Containers = append(pod.Spec.Containers, *container)
 
+	kubemanifest.MarkPodAsCritical(pod)
+
 	return pod, nil
 }

pkg/kubemanifest/critical.go

@@ -0,0 +1,35 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kubemanifest
+
+import (
+	"k8s.io/client-go/pkg/api/v1"
+)
+
+// MarkPodAsCritical adds the required annotations for a pod to be considered critical
+func MarkPodAsCritical(pod *v1.Pod) {
+	if pod.ObjectMeta.Annotations == nil {
+		pod.ObjectMeta.Annotations = make(map[string]string)
+	}
+	pod.ObjectMeta.Annotations["scheduler.alpha.kubernetes.io/critical-pod"] = ""
+
+	toleration := v1.Toleration{
+		Key:      "CriticalAddonsOnly",
+		Operator: v1.TolerationOpExists,
+	}
+	pod.Spec.Tolerations = append(pod.Spec.Tolerations, toleration)
+}

protokube/pkg/protokube/etcd_manifest.go

@@ -23,6 +23,7 @@ import (
 
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/client-go/pkg/api/v1"
+	"k8s.io/kops/pkg/kubemanifest"
 )
 
 // BuildEtcdManifest creates the pod spec, based on the etcd cluster
@@ -142,6 +143,8 @@ func BuildEtcdManifest(c *EtcdCluster) *v1.Pod {
 		pod.Spec.Containers = append(pod.Spec.Containers, container)
 	}
 
+	kubemanifest.MarkPodAsCritical(pod)
+
 	return pod
 }
 

protokube/tests/integration/build_etcd_manifest/main/non_tls.yaml

@@ -23,6 +23,8 @@ spec: {}
 apiVersion: v1
 kind: Pod
 metadata:
+  annotations:
+    scheduler.alpha.kubernetes.io/critical-pod: ""
   creationTimestamp: null
   labels:
     k8s-app: etcd-server-main
@@ -82,6 +84,9 @@ spec:
       name: hosts
       readOnly: true
   hostNetwork: true
+  tolerations:
+  - key: CriticalAddonsOnly
+    operator: Exists
   volumes:
   - hostPath:
       path: /mnt/main/var/etcd/data-main

protokube/tests/integration/build_etcd_manifest/main/tls.yaml

@@ -29,6 +29,8 @@ spec: {}
 apiVersion: v1
 kind: Pod
 metadata:
+  annotations:
+    scheduler.alpha.kubernetes.io/critical-pod: ""
   creationTimestamp: null
   labels:
     k8s-app: etcd-server-main
@@ -101,6 +103,9 @@ spec:
       name: srvkubernetes
       readOnly: true
   hostNetwork: true
+  tolerations:
+  - key: CriticalAddonsOnly
+    operator: Exists
   volumes:
   - hostPath:
       path: /mnt/main/var/etcd/data-main