Merge pull request #93947 from mgugino-upstream-stage/fix-drain-forbidden

Fix drain forbidden

Kubernetes-commit: f96debad10f8deddab9c8e9a42da1e16e790b068
This commit is contained in:
Kubernetes Publisher 2020-09-03 04:55:41 -07:00
commit e31d0f0d20
4 changed files with 42 additions and 18 deletions

8
Godeps/Godeps.json generated
View File

@ -760,11 +760,11 @@
},
{
"ImportPath": "k8s.io/api",
"Rev": "73d7eb3bb026"
"Rev": "03aa42fe49ac"
},
{
"ImportPath": "k8s.io/apimachinery",
"Rev": "ba0f2f062330"
"Rev": "6084ad015ddd"
},
{
"ImportPath": "k8s.io/cli-runtime",
@ -772,11 +772,11 @@
},
{
"ImportPath": "k8s.io/client-go",
"Rev": "b643ec487eb7"
"Rev": "ba26c4506d39"
},
{
"ImportPath": "k8s.io/code-generator",
"Rev": "5a311e69ffcf"
"Rev": "7dbebec02af3"
},
{
"ImportPath": "k8s.io/component-base",

14
go.mod
View File

@ -34,10 +34,10 @@ require (
github.com/xlab/handysort v0.0.0-20150421192137-fb3537ed64a1 // indirect
golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4
gopkg.in/yaml.v2 v2.2.8
k8s.io/api v0.0.0-20200902051604-73d7eb3bb026
k8s.io/apimachinery v0.0.0-20200902131538-ba0f2f062330
k8s.io/api v0.0.0-20200902131916-03aa42fe49ac
k8s.io/apimachinery v0.0.0-20200902171538-6084ad015ddd
k8s.io/cli-runtime v0.0.0-20200902135640-4d285f01f90f
k8s.io/client-go v0.0.0-20200902132332-b643ec487eb7
k8s.io/client-go v0.0.0-20200903011833-ba26c4506d39
k8s.io/component-base v0.0.0-20200902133139-1d2c8187d823
k8s.io/klog/v2 v2.2.0
k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6
@ -49,11 +49,11 @@ require (
)
replace (
k8s.io/api => k8s.io/api v0.0.0-20200902051604-73d7eb3bb026
k8s.io/apimachinery => k8s.io/apimachinery v0.0.0-20200902131538-ba0f2f062330
k8s.io/api => k8s.io/api v0.0.0-20200902131916-03aa42fe49ac
k8s.io/apimachinery => k8s.io/apimachinery v0.0.0-20200902171538-6084ad015ddd
k8s.io/cli-runtime => k8s.io/cli-runtime v0.0.0-20200902135640-4d285f01f90f
k8s.io/client-go => k8s.io/client-go v0.0.0-20200902132332-b643ec487eb7
k8s.io/code-generator => k8s.io/code-generator v0.0.0-20200813011144-5a311e69ffcf
k8s.io/client-go => k8s.io/client-go v0.0.0-20200903011833-ba26c4506d39
k8s.io/code-generator => k8s.io/code-generator v0.0.0-20200902211251-7dbebec02af3
k8s.io/component-base => k8s.io/component-base v0.0.0-20200902133139-1d2c8187d823
k8s.io/metrics => k8s.io/metrics v0.0.0-20200902135418-05baf84b7531
)

8
go.sum
View File

@ -503,11 +503,11 @@ honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWh
honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
k8s.io/api v0.0.0-20200902051604-73d7eb3bb026/go.mod h1:qMn0Ckwm7+s/YbS3iDBlHErlW/SeOXttpV83LV5iO5M=
k8s.io/apimachinery v0.0.0-20200902131538-ba0f2f062330/go.mod h1:DnPGDnARWFvYa3pMHgSxtbZb7gpzzAZ1pTfaUNDVlmA=
k8s.io/api v0.0.0-20200902131916-03aa42fe49ac/go.mod h1:RRBubtUUfQknZlqyrIcyOzchPl/ndgo1xAGCzjrKrQU=
k8s.io/apimachinery v0.0.0-20200902171538-6084ad015ddd/go.mod h1:DnPGDnARWFvYa3pMHgSxtbZb7gpzzAZ1pTfaUNDVlmA=
k8s.io/cli-runtime v0.0.0-20200902135640-4d285f01f90f/go.mod h1:dMX648DBrr+l8yWksbYdwcMzDEM/H0QGyJ8EOKiF9xY=
k8s.io/client-go v0.0.0-20200902132332-b643ec487eb7/go.mod h1:1CDKw+WUI72vWcFo0SpATCUkR78esmyqaIDImRWHmkk=
k8s.io/code-generator v0.0.0-20200813011144-5a311e69ffcf/go.mod h1:moqLn7w0t9cMs4+5CQyxnfA/HV8MF6aAVENF+WZZhgk=
k8s.io/client-go v0.0.0-20200903011833-ba26c4506d39/go.mod h1:IlEd8lxjDG8cvAB6MHAPJe9pO2IOxBafW1hrpyMFf0U=
k8s.io/code-generator v0.0.0-20200902211251-7dbebec02af3/go.mod h1:moqLn7w0t9cMs4+5CQyxnfA/HV8MF6aAVENF+WZZhgk=
k8s.io/component-base v0.0.0-20200902133139-1d2c8187d823/go.mod h1:0gnPCcp/GY+eCzs3ctBHW+6CDRXi6+WyHI39AhX3nHQ=
k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
k8s.io/gengo v0.0.0-20200428234225-8167cfdcfc14/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=

View File

@ -254,6 +254,7 @@ func (d *Helper) evictPods(pods []corev1.Pod, policyGroupVersion string, getPodF
defer cancel()
for _, pod := range pods {
go func(pod corev1.Pod, returnCh chan error) {
refreshPod := false
for {
switch d.DryRunStrategy {
case cmdutil.DryRunServer:
@ -268,17 +269,40 @@ func (d *Helper) evictPods(pods []corev1.Pod, policyGroupVersion string, getPodF
return
default:
}
err := d.EvictPod(pod, policyGroupVersion)
// Create a temporary pod so we don't mutate the pod in the loop.
activePod := pod
if refreshPod {
freshPod, err := getPodFn(pod.Namespace, pod.Name)
// We ignore errors and let eviction sort it out with
// the original pod.
if err == nil {
activePod = *freshPod
}
refreshPod = false
}
err := d.EvictPod(activePod, policyGroupVersion)
if err == nil {
break
} else if apierrors.IsNotFound(err) {
returnCh <- nil
return
} else if apierrors.IsTooManyRequests(err) {
fmt.Fprintf(d.ErrOut, "error when evicting pods/%q -n %q (will retry after 5s): %v\n", pod.Name, pod.Namespace, err)
fmt.Fprintf(d.ErrOut, "error when evicting pods/%q -n %q (will retry after 5s): %v\n", activePod.Name, activePod.Namespace, err)
time.Sleep(5 * time.Second)
} else if !activePod.ObjectMeta.DeletionTimestamp.IsZero() && apierrors.IsForbidden(err) && apierrors.HasStatusCause(err, corev1.NamespaceTerminatingCause) {
// an eviction request in a deleting namespace will throw a forbidden error,
// if the pod is already marked deleted, we can ignore this error, an eviction
// request will never succeed, but we will waitForDelete for this pod.
break
} else if apierrors.IsForbidden(err) && apierrors.HasStatusCause(err, corev1.NamespaceTerminatingCause) {
// an eviction request in a deleting namespace will throw a forbidden error,
// if the pod is not marked deleted, we retry until it is.
fmt.Fprintf(d.ErrOut, "error when evicting pod %q (will retry after 5s): %v\n", activePod.Name, err)
time.Sleep(5 * time.Second)
} else {
returnCh <- fmt.Errorf("error when evicting pods/%q -n %q: %v", pod.Name, pod.Namespace, err)
returnCh <- fmt.Errorf("error when evicting pods/%q -n %q: %v", activePod.Name, activePod.Namespace, err)
return
}
}