upup: delete cluster should eventually give up

If it isn't making progress, eventually we should bail out so that we
can use this in scripts.  However, we don't need to be too aggressive,
because it is not good to leak resources, and a user will likely
Control-C us pretty fast when they see they're in a loop.

Issue #99
This commit is contained in:
Justin Santa Barbara 2016-06-11 21:17:02 -04:00
parent 4e744ffa32
commit e63426ce65
1 changed files with 8 additions and 0 deletions

View File

@ -194,6 +194,7 @@ func (c *DeleteCluster) DeleteResources(resources map[string]DeletableResource)
glog.Infof("\t%s\t%v", k, v)
}
iterationsWithNoProgress := 0
for {
// TODO: Some form of default ordering based on types?
// TODO: Give up eventually?
@ -258,6 +259,7 @@ func (c *DeleteCluster) DeleteResources(resources map[string]DeletableResource)
fmt.Printf("%s\tok\n", k)
delete(failed, k)
iterationsWithNoProgress = 0
done[k] = r
mutex.Unlock()
}
@ -278,6 +280,12 @@ func (c *DeleteCluster) DeleteResources(resources map[string]DeletableResource)
fmt.Printf("\t%s\n", k)
}
iterationsWithNoProgress++
if iterationsWithNoProgress > 30 {
return fmt.Errorf("Not making progress deleting resources; giving up")
}
time.Sleep(10 * time.Second)
}
}