From 383194780a15cc8727e6400c5302c5adabadfaac Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Sat, 30 Sep 2017 15:01:19 -0400 Subject: [PATCH] Create helper function for critical pod annotations In particularly I think we want a toleration also; easiest to put the code in one function. --- nodeup/pkg/model/kube_apiserver.go | 4 ++- nodeup/pkg/model/kube_controller_manager.go | 6 ++-- nodeup/pkg/model/kube_proxy.go | 16 ++++----- nodeup/pkg/model/kube_scheduler.go | 6 ++-- pkg/kubemanifest/critical.go | 35 +++++++++++++++++++ protokube/pkg/protokube/etcd_manifest.go | 3 ++ .../build_etcd_manifest/main/non_tls.yaml | 5 +++ .../build_etcd_manifest/main/tls.yaml | 5 +++ 8 files changed, 65 insertions(+), 15 deletions(-) create mode 100644 pkg/kubemanifest/critical.go diff --git a/nodeup/pkg/model/kube_apiserver.go b/nodeup/pkg/model/kube_apiserver.go index 3906647057..f173355051 100644 --- a/nodeup/pkg/model/kube_apiserver.go +++ b/nodeup/pkg/model/kube_apiserver.go @@ -31,6 +31,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/client-go/pkg/api/v1" + "k8s.io/kops/pkg/kubemanifest" ) const PathAuthnConfig = "/etc/kubernetes/authn.config" @@ -302,6 +303,8 @@ func (b *KubeAPIServerBuilder) buildPod() (*v1.Pod, error) { pod.Spec.Containers = append(pod.Spec.Containers, *container) + kubemanifest.MarkPodAsCritical(pod) + return pod, nil } @@ -311,7 +314,6 @@ func (b *KubeAPIServerBuilder) buildAnnotations() map[string]string { if b.Cluster.Spec.API != nil && b.Cluster.Spec.API.DNS != nil { annotations["dns.alpha.kubernetes.io/external"] = b.Cluster.Spec.MasterPublicName } - annotations["scheduler.alpha.kubernetes.io/critical-pod"] = "" return annotations } diff --git a/nodeup/pkg/model/kube_controller_manager.go b/nodeup/pkg/model/kube_controller_manager.go index e4f113b7ba..bb20eaa56e 100644 --- a/nodeup/pkg/model/kube_controller_manager.go +++ b/nodeup/pkg/model/kube_controller_manager.go @@ -29,6 +29,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/client-go/pkg/api/v1" + "k8s.io/kops/pkg/kubemanifest" ) // KubeControllerManagerBuilder install kube-controller-manager (just the manifest at the moment) @@ -149,9 +150,6 @@ func (b *KubeControllerManagerBuilder) buildPod() (*v1.Pod, error) { ObjectMeta: metav1.ObjectMeta{ Name: "kube-controller-manager", Namespace: "kube-system", - Annotations: map[string]string{ - "scheduler.alpha.kubernetes.io/critical-pod": "", - }, Labels: map[string]string{ "k8s-app": "kube-controller-manager", }, @@ -207,5 +205,7 @@ func (b *KubeControllerManagerBuilder) buildPod() (*v1.Pod, error) { pod.Spec.Containers = append(pod.Spec.Containers, *container) + kubemanifest.MarkPodAsCritical(pod) + return pod, nil } diff --git a/nodeup/pkg/model/kube_proxy.go b/nodeup/pkg/model/kube_proxy.go index 5e11b8d9e8..97c73407bd 100644 --- a/nodeup/pkg/model/kube_proxy.go +++ b/nodeup/pkg/model/kube_proxy.go @@ -29,6 +29,7 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/pkg/api/v1" + "k8s.io/kops/pkg/kubemanifest" ) // KubeProxyBuilder installs kube-proxy @@ -158,14 +159,6 @@ func (b *KubeProxyBuilder) buildPod() (*v1.Pod, error) { "k8s-app": "kube-proxy", "tier": "node", }, - Annotations: map[string]string{ - // This annotation ensures that kube-proxy does not get evicted if the node - // supports critical pod annotation based priority scheme. - // Note that kube-proxy runs as a static pod so this annotation does NOT have - // any effect on rescheduler (default scheduler and rescheduler are not - // involved in scheduling kube-proxy). - "scheduler.alpha.kubernetes.io/critical-pod": "", - }, }, Spec: v1.PodSpec{ HostNetwork: true, @@ -210,6 +203,13 @@ func (b *KubeProxyBuilder) buildPod() (*v1.Pod, error) { // }, //} + // This annotation ensures that kube-proxy does not get evicted if the node + // supports critical pod annotation based priority scheme. + // Note that kube-proxy runs as a static pod so this annotation does NOT have + // any effect on rescheduler (default scheduler and rescheduler are not + // involved in scheduling kube-proxy). + kubemanifest.MarkPodAsCritical(pod) + return pod, nil } diff --git a/nodeup/pkg/model/kube_scheduler.go b/nodeup/pkg/model/kube_scheduler.go index e9dfcdfff7..72b7fe98bd 100644 --- a/nodeup/pkg/model/kube_scheduler.go +++ b/nodeup/pkg/model/kube_scheduler.go @@ -28,6 +28,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/client-go/pkg/api/v1" + "k8s.io/kops/pkg/kubemanifest" ) // KubeSchedulerBuilder install kube-scheduler @@ -110,9 +111,6 @@ func (b *KubeSchedulerBuilder) buildPod() (*v1.Pod, error) { Labels: map[string]string{ "k8s-app": "kube-scheduler", }, - Annotations: map[string]string{ - "scheduler.alpha.kubernetes.io/critical-pod": "", - }, }, Spec: v1.PodSpec{ HostNetwork: true, @@ -149,5 +147,7 @@ func (b *KubeSchedulerBuilder) buildPod() (*v1.Pod, error) { pod.Spec.Containers = append(pod.Spec.Containers, *container) + kubemanifest.MarkPodAsCritical(pod) + return pod, nil } diff --git a/pkg/kubemanifest/critical.go b/pkg/kubemanifest/critical.go new file mode 100644 index 0000000000..4246827f9b --- /dev/null +++ b/pkg/kubemanifest/critical.go @@ -0,0 +1,35 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubemanifest + +import ( + "k8s.io/client-go/pkg/api/v1" +) + +// MarkPodAsCritical adds the required annotations for a pod to be considered critical +func MarkPodAsCritical(pod *v1.Pod) { + if pod.ObjectMeta.Annotations == nil { + pod.ObjectMeta.Annotations = make(map[string]string) + } + pod.ObjectMeta.Annotations["scheduler.alpha.kubernetes.io/critical-pod"] = "" + + toleration := v1.Toleration{ + Key: "CriticalAddonsOnly", + Operator: v1.TolerationOpExists, + } + pod.Spec.Tolerations = append(pod.Spec.Tolerations, toleration) +} diff --git a/protokube/pkg/protokube/etcd_manifest.go b/protokube/pkg/protokube/etcd_manifest.go index 0af76cc1fe..b8ea5aa7fb 100644 --- a/protokube/pkg/protokube/etcd_manifest.go +++ b/protokube/pkg/protokube/etcd_manifest.go @@ -23,6 +23,7 @@ import ( "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/client-go/pkg/api/v1" + "k8s.io/kops/pkg/kubemanifest" ) // BuildEtcdManifest creates the pod spec, based on the etcd cluster @@ -142,6 +143,8 @@ func BuildEtcdManifest(c *EtcdCluster) *v1.Pod { pod.Spec.Containers = append(pod.Spec.Containers, container) } + kubemanifest.MarkPodAsCritical(pod) + return pod } diff --git a/protokube/tests/integration/build_etcd_manifest/main/non_tls.yaml b/protokube/tests/integration/build_etcd_manifest/main/non_tls.yaml index 7c91fc1a38..a04339522f 100644 --- a/protokube/tests/integration/build_etcd_manifest/main/non_tls.yaml +++ b/protokube/tests/integration/build_etcd_manifest/main/non_tls.yaml @@ -23,6 +23,8 @@ spec: {} apiVersion: v1 kind: Pod metadata: + annotations: + scheduler.alpha.kubernetes.io/critical-pod: "" creationTimestamp: null labels: k8s-app: etcd-server-main @@ -82,6 +84,9 @@ spec: name: hosts readOnly: true hostNetwork: true + tolerations: + - key: CriticalAddonsOnly + operator: Exists volumes: - hostPath: path: /mnt/main/var/etcd/data-main diff --git a/protokube/tests/integration/build_etcd_manifest/main/tls.yaml b/protokube/tests/integration/build_etcd_manifest/main/tls.yaml index 6a5cbc7537..cd082f9293 100644 --- a/protokube/tests/integration/build_etcd_manifest/main/tls.yaml +++ b/protokube/tests/integration/build_etcd_manifest/main/tls.yaml @@ -29,6 +29,8 @@ spec: {} apiVersion: v1 kind: Pod metadata: + annotations: + scheduler.alpha.kubernetes.io/critical-pod: "" creationTimestamp: null labels: k8s-app: etcd-server-main @@ -101,6 +103,9 @@ spec: name: srvkubernetes readOnly: true hostNetwork: true + tolerations: + - key: CriticalAddonsOnly + operator: Exists volumes: - hostPath: path: /mnt/main/var/etcd/data-main