validate cluster twice

cmd/kops/rollingupdatecluster.go

@@ -415,6 +415,7 @@ func RunRollingUpdateCluster(f *util.Factory, out io.Writer, options *RollingUpd
 		ValidationTimeout: options.ValidationTimeout,
 		// TODO should we expose this to the UI?
 		ValidateTickDuration:    30 * time.Second,
+		ValidateSuccessDuration: 10 * time.Second,
 	}
 	return d.RollingUpdate(groups, cluster, list)
 }

pkg/instancegroups/instancegroups.go

@@ -262,9 +262,16 @@ func (r *RollingUpdateInstanceGroup) tryValidateCluster(rollingUpdateData *Rolli
 		klog.Infof("Cluster did not pass validation, will try again in %q until duration %q expires: %s.", tickDuration, duration, strings.Join(messages, ", "))
 		return false
 	} else {
+		klog.Info("Cluster validated, revalidating to make sure it does not flap.")
+		time.Sleep(rollingUpdateData.ValidateSuccessDuration)
+		result, err = rollingUpdateData.ClusterValidator.Validate()
+		if err == nil && len(result.Failures) == 0 {
 			klog.Info("Cluster validated.")
 			return true
 		}
+		klog.Info("Cluster did not revalidate.")
+		return false
+	}
 }
 
 // validateCluster runs our validation methods on the K8s Cluster.

pkg/instancegroups/rollingupdate.go

@@ -63,6 +63,10 @@ type RollingUpdateCluster struct {
 
 	// ValidateTickDuration is the amount of time to wait between cluster validation attempts
 	ValidateTickDuration time.Duration
+
+	// ValidateSuccessDuration is the amount of time a cluster must continue to validate successfully
+	// before updating the next node
+	ValidateSuccessDuration time.Duration
 }
 
 // RollingUpdate performs a rolling update on a K8s Cluster.

pkg/instancegroups/rollingupdate_test.go

@@ -55,6 +55,7 @@ func getTestSetup() (*RollingUpdateCluster, awsup.AWSCloud, *kopsapi.Cluster) {
 		ClusterValidator:        &successfulClusterValidator{},
 		FailOnValidate:          true,
 		ValidateTickDuration:    1 * time.Millisecond,
+		ValidateSuccessDuration: 5 * time.Millisecond,
 	}
 
 	return c, mockcloud, cluster