Stop with rolling update if bastions or masters failed to update

This commit is contained in:
Deniz Zoeteman 2018-07-16 16:54:11 +02:00
parent 498b5d72b6
commit b06e3efa4d
1 changed files with 28 additions and 1 deletions

View File

@ -111,6 +111,13 @@ func (c *RollingUpdateCluster) RollingUpdate(groups map[string]*cloudinstances.C
wg.Wait()
}
// Do not continue update if bastion(s) failed
for _, err := range results {
if err != nil {
return err
}
}
// Upgrade master next
{
var wg sync.WaitGroup
@ -135,17 +142,37 @@ func (c *RollingUpdateCluster) RollingUpdate(groups map[string]*cloudinstances.C
err = g.RollingUpdate(c, cluster, instanceGroups, false, c.MasterInterval, c.ValidationTimeout)
}
if err != nil {
// Remove function panic errors if an actual error occurred, otherwise that would be displayed
for k := range masterGroups {
resultsMutex.Lock()
results[k] = nil
resultsMutex.Unlock()
}
}
resultsMutex.Lock()
results[k] = err
resultsMutex.Unlock()
// TODO: Bail on error?
if err != nil {
// Stop before all masters are updated if an error occurs,
// to prevent more masters from failing and causing an outage
return
}
}
}()
wg.Wait()
}
// Do not continue update if master(s) failed, cluster is potentially in an unhealthy state
for _, err := range results {
if err != nil {
return err
}
}
// Upgrade nodes, with greater parallelism
{
var wg sync.WaitGroup