kubernetes
/
kubectl
mirror of https://github.com/kubernetes/kubectl.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Handle eviction of pods in deleted namespace 

If a pod is already marked deleted, and the eviction
api returns an unauthorized response, ignore that
error since the pod is marked for deletion already.

If the pod is not already marked deleted, retry.

Kubernetes-commit: 8d2a2ffe014ed06b5f8971e7f3dc25ec155e44d4
This commit is contained in:
Michael Gugino 2020-08-12 16:15:54 -04:00 committed by Kubernetes Publisher
parent de06b92a12
commit e5ddedb459
1 changed files with 26 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
pkg/drain/drain.go
View File

@ -254,6 +254,7 @@ func (d *Helper) evictPods(pods []corev1.Pod, policyGroupVersion string, getPodF
defer cancel()
for _, pod := range pods {
go func(pod corev1.Pod, returnCh chan error) {
refreshPod := false
for {
switch d.DryRunStrategy {
case cmdutil.DryRunServer:
@ -268,20 +269,40 @@ func (d *Helper) evictPods(pods []corev1.Pod, policyGroupVersion string, getPodF
return
default:
}
err := d.EvictPod(pod, policyGroupVersion)
// Create a temporary pod so we don't mutate the pod in the loop.
activePod := pod
if refreshPod {
freshPod, err := getPodFn(pod.Namespace, pod.Name)
// We ignore errors and let eviction sort it out with
// the original pod.
if err == nil {
activePod = *freshPod
}
refreshPod = false
}
err := d.EvictPod(activePod, policyGroupVersion)
if err == nil {
break
} else if apierrors.IsNotFound(err) {
returnCh <- nil
return
} else if apierrors.IsTooManyRequests(err) {
fmt.Fprintf(d.ErrOut, "error when evicting pods/%q -n %q (will retry after 5s): %v\n", pod.Name, pod.Namespace, err)
fmt.Fprintf(d.ErrOut, "error when evicting pods/%q -n %q (will retry after 5s): %v\n", activePod.Name, activePod.Namespace, err)
time.Sleep(5 * time.Second)
} else if apierrors.IsForbidden(err) {
// an eviction request in a deleting namespace will throw a forbidden error
} else if !activePod.ObjectMeta.DeletionTimestamp.IsZero() && apierrors.IsForbidden(err) && apierrors.HasStatusCause(err, corev1.NamespaceTerminatingCause) {
// an eviction request in a deleting namespace will throw a forbidden error,
// if the pod is already marked deleted, we can ignore this error, an eviction
// request will never succeed, but we will waitForDelete for this pod.
break
} else if apierrors.IsForbidden(err) && apierrors.HasStatusCause(err, corev1.NamespaceTerminatingCause) {
// an eviction request in a deleting namespace will throw a forbidden error,
// if the pod is not marked deleted, we retry until it is.
fmt.Fprintf(d.ErrOut, "error when evicting pod %q (will retry after 5s): %v\n", activePod.Name, err)
time.Sleep(5 * time.Second)
} else {
returnCh <- fmt.Errorf("error when evicting pods/%q -n %q: %v", pod.Name, pod.Namespace, err)
returnCh <- fmt.Errorf("error when evicting pods/%q -n %q: %v", activePod.Name, activePod.Namespace, err)
return
}
}