diff --git a/cmd/kops/rollingupdatecluster.go b/cmd/kops/rollingupdatecluster.go index e61de2cd6c..d5f1d0e8ef 100644 --- a/cmd/kops/rollingupdatecluster.go +++ b/cmd/kops/rollingupdatecluster.go @@ -380,5 +380,5 @@ func RunRollingUpdateCluster(f *util.Factory, out io.Writer, options *RollingUpd PostDrainDelay: options.PostDrainDelay, ValidationTimeout: options.ValidationTimeout, } - return d.RollingUpdate(groups, list) + return d.RollingUpdate(groups, cluster, list) } diff --git a/cmd/kops/validate_cluster.go b/cmd/kops/validate_cluster.go index 36eeb1accb..43239ea794 100644 --- a/cmd/kops/validate_cluster.go +++ b/cmd/kops/validate_cluster.go @@ -161,7 +161,7 @@ func RunValidateCluster(f *util.Factory, cmd *cobra.Command, args []string, out } } - validationCluster, validationFailed := validation.ValidateCluster(cluster.ObjectMeta.Name, list, k8sClient) + validationCluster, validationFailed := validation.ValidateCluster(cluster, list, k8sClient) if validationCluster == nil || validationCluster.NodeList == nil || validationCluster.NodeList.Items == nil { return validationFailed diff --git a/pkg/instancegroups/instancegroups.go b/pkg/instancegroups/instancegroups.go index 90a3e910b7..bc4ae242b8 100644 --- a/pkg/instancegroups/instancegroups.go +++ b/pkg/instancegroups/instancegroups.go @@ -97,7 +97,7 @@ func promptInteractive(upgradedHost string) (stopPrompting bool, err error) { // TODO: Batch termination, like a rolling-update // RollingUpdate performs a rolling update on a list of ec2 instances. -func (r *RollingUpdateInstanceGroup) RollingUpdate(rollingUpdateData *RollingUpdateCluster, instanceGroupList *api.InstanceGroupList, isBastion bool, sleepAfterTerminate time.Duration, validationTimeout time.Duration) (err error) { +func (r *RollingUpdateInstanceGroup) RollingUpdate(rollingUpdateData *RollingUpdateCluster, cluster *api.Cluster, instanceGroupList *api.InstanceGroupList, isBastion bool, sleepAfterTerminate time.Duration, validationTimeout time.Duration) (err error) { // we should not get here, but hey I am going to check. if rollingUpdateData == nil { @@ -127,7 +127,7 @@ func (r *RollingUpdateInstanceGroup) RollingUpdate(rollingUpdateData *RollingUpd } else if rollingUpdateData.CloudOnly { glog.V(3).Info("Not validating cluster as validation is turned off via the cloud-only flag.") } else if featureflag.DrainAndValidateRollingUpdate.Enabled() { - if err = r.ValidateCluster(rollingUpdateData, instanceGroupList); err != nil { + if err = r.ValidateCluster(rollingUpdateData, cluster, instanceGroupList); err != nil { if rollingUpdateData.FailOnValidate { return fmt.Errorf("error validating cluster: %v", err) } else { @@ -187,7 +187,7 @@ func (r *RollingUpdateInstanceGroup) RollingUpdate(rollingUpdateData *RollingUpd } else if featureflag.DrainAndValidateRollingUpdate.Enabled() { glog.Infof("Validating the cluster.") - if err = r.ValidateClusterWithDuration(rollingUpdateData, instanceGroupList, validationTimeout); err != nil { + if err = r.ValidateClusterWithDuration(rollingUpdateData, cluster, instanceGroupList, validationTimeout); err != nil { if rollingUpdateData.FailOnValidate { glog.Errorf("Cluster did not validate within %s", validationTimeout) @@ -213,12 +213,12 @@ func (r *RollingUpdateInstanceGroup) RollingUpdate(rollingUpdateData *RollingUpd } // ValidateClusterWithDuration runs validation.ValidateCluster until either we get positive result or the timeout expires -func (r *RollingUpdateInstanceGroup) ValidateClusterWithDuration(rollingUpdateData *RollingUpdateCluster, instanceGroupList *api.InstanceGroupList, duration time.Duration) error { +func (r *RollingUpdateInstanceGroup) ValidateClusterWithDuration(rollingUpdateData *RollingUpdateCluster, cluster *api.Cluster, instanceGroupList *api.InstanceGroupList, duration time.Duration) error { // TODO should we expose this to the UI? tickDuration := 30 * time.Second // Try to validate cluster at least once, this will handle durations that are lower // than our tick time - if r.tryValidateCluster(rollingUpdateData, instanceGroupList, duration, tickDuration) { + if r.tryValidateCluster(rollingUpdateData, cluster, instanceGroupList, duration, tickDuration) { return nil } @@ -232,7 +232,7 @@ func (r *RollingUpdateInstanceGroup) ValidateClusterWithDuration(rollingUpdateDa return fmt.Errorf("cluster did not validate within a duation of %q", duration) case <-tick: // Got a tick, validate cluster - if r.tryValidateCluster(rollingUpdateData, instanceGroupList, duration, tickDuration) { + if r.tryValidateCluster(rollingUpdateData, cluster, instanceGroupList, duration, tickDuration) { return nil } // ValidateCluster didn't work yet, so let's try again @@ -241,8 +241,8 @@ func (r *RollingUpdateInstanceGroup) ValidateClusterWithDuration(rollingUpdateDa } } -func (r *RollingUpdateInstanceGroup) tryValidateCluster(rollingUpdateData *RollingUpdateCluster, instanceGroupList *api.InstanceGroupList, duration time.Duration, tickDuration time.Duration) bool { - if _, err := validation.ValidateCluster(rollingUpdateData.ClusterName, instanceGroupList, rollingUpdateData.K8sClient); err != nil { +func (r *RollingUpdateInstanceGroup) tryValidateCluster(rollingUpdateData *RollingUpdateCluster, cluster *api.Cluster, instanceGroupList *api.InstanceGroupList, duration time.Duration, tickDuration time.Duration) bool { + if _, err := validation.ValidateCluster(cluster, instanceGroupList, rollingUpdateData.K8sClient); err != nil { glog.Infof("Cluster did not validate, will try again in %q until duration %q expires: %v.", tickDuration, duration, err) return false } else { @@ -252,10 +252,9 @@ func (r *RollingUpdateInstanceGroup) tryValidateCluster(rollingUpdateData *Rolli } // ValidateCluster runs our validation methods on the K8s Cluster. -func (r *RollingUpdateInstanceGroup) ValidateCluster(rollingUpdateData *RollingUpdateCluster, instanceGroupList *api.InstanceGroupList) error { - - if _, err := validation.ValidateCluster(rollingUpdateData.ClusterName, instanceGroupList, rollingUpdateData.K8sClient); err != nil { - return fmt.Errorf("cluster %q did not pass validation: %v", rollingUpdateData.ClusterName, err) +func (r *RollingUpdateInstanceGroup) ValidateCluster(rollingUpdateData *RollingUpdateCluster, cluster *api.Cluster, instanceGroupList *api.InstanceGroupList) error { + if _, err := validation.ValidateCluster(cluster, instanceGroupList, rollingUpdateData.K8sClient); err != nil { + return fmt.Errorf("cluster %q did not pass validation: %v", cluster.Name, err) } return nil diff --git a/pkg/instancegroups/rollingupdate.go b/pkg/instancegroups/rollingupdate.go index 9ac66941e5..f57c9147c2 100644 --- a/pkg/instancegroups/rollingupdate.go +++ b/pkg/instancegroups/rollingupdate.go @@ -59,7 +59,7 @@ type RollingUpdateCluster struct { } // RollingUpdate performs a rolling update on a K8s Cluster. -func (c *RollingUpdateCluster) RollingUpdate(groups map[string]*cloudinstances.CloudInstanceGroup, instanceGroups *api.InstanceGroupList) error { +func (c *RollingUpdateCluster) RollingUpdate(groups map[string]*cloudinstances.CloudInstanceGroup, cluster *api.Cluster, instanceGroups *api.InstanceGroupList) error { if len(groups) == 0 { glog.Infof("Cloud Instance Group length is zero. Not doing a rolling-update.") return nil @@ -99,7 +99,7 @@ func (c *RollingUpdateCluster) RollingUpdate(groups map[string]*cloudinstances.C g, err := NewRollingUpdateInstanceGroup(c.Cloud, group) if err == nil { - err = g.RollingUpdate(c, instanceGroups, true, c.BastionInterval, c.ValidationTimeout) + err = g.RollingUpdate(c, cluster, instanceGroups, true, c.BastionInterval, c.ValidationTimeout) } resultsMutex.Lock() @@ -132,7 +132,7 @@ func (c *RollingUpdateCluster) RollingUpdate(groups map[string]*cloudinstances.C for k, group := range masterGroups { g, err := NewRollingUpdateInstanceGroup(c.Cloud, group) if err == nil { - err = g.RollingUpdate(c, instanceGroups, false, c.MasterInterval, c.ValidationTimeout) + err = g.RollingUpdate(c, cluster, instanceGroups, false, c.MasterInterval, c.ValidationTimeout) } resultsMutex.Lock() @@ -170,7 +170,7 @@ func (c *RollingUpdateCluster) RollingUpdate(groups map[string]*cloudinstances.C for k, group := range nodeGroups { g, err := NewRollingUpdateInstanceGroup(c.Cloud, group) if err == nil { - err = g.RollingUpdate(c, instanceGroups, false, c.NodeInterval, c.ValidationTimeout) + err = g.RollingUpdate(c, cluster, instanceGroups, false, c.NodeInterval, c.ValidationTimeout) } resultsMutex.Lock() diff --git a/pkg/instancegroups/rollingupdate_test.go b/pkg/instancegroups/rollingupdate_test.go index 71a40d06c2..77ebebb7b2 100644 --- a/pkg/instancegroups/rollingupdate_test.go +++ b/pkg/instancegroups/rollingupdate_test.go @@ -85,6 +85,9 @@ func TestRollingUpdateAllNeedUpdate(t *testing.T) { mockcloud := awsup.BuildMockAWSCloud("us-east-1", "abc") mockcloud.MockAutoscaling = &mockautoscaling.MockAutoscaling{} + cluster := &kopsapi.Cluster{} + cluster.Name = "test.k8s.local" + c := &RollingUpdateCluster{ Cloud: mockcloud, MasterInterval: 1 * time.Millisecond, @@ -207,7 +210,7 @@ func TestRollingUpdateAllNeedUpdate(t *testing.T) { }, } - err := c.RollingUpdate(groups, &kopsapi.InstanceGroupList{}) + err := c.RollingUpdate(groups, cluster, &kopsapi.InstanceGroupList{}) if err != nil { t.Errorf("Error on rolling update: %v", err) } @@ -226,6 +229,9 @@ func TestRollingUpdateNoneNeedUpdate(t *testing.T) { mockcloud := awsup.BuildMockAWSCloud("us-east-1", "abc") mockcloud.MockAutoscaling = &mockautoscaling.MockAutoscaling{} + cluster := &kopsapi.Cluster{} + cluster.Name = "test.k8s.local" + c := &RollingUpdateCluster{ Cloud: mockcloud, MasterInterval: 1 * time.Millisecond, @@ -317,7 +323,7 @@ func TestRollingUpdateNoneNeedUpdate(t *testing.T) { }, } - err := c.RollingUpdate(groups, &kopsapi.InstanceGroupList{}) + err := c.RollingUpdate(groups, cluster, &kopsapi.InstanceGroupList{}) if err != nil { t.Errorf("Error on rolling update: %v", err) } @@ -365,6 +371,9 @@ func TestRollingUpdateNoneNeedUpdateWithForce(t *testing.T) { mockcloud := awsup.BuildMockAWSCloud("us-east-1", "abc") mockcloud.MockAutoscaling = &mockautoscaling.MockAutoscaling{} + cluster := &kopsapi.Cluster{} + cluster.Name = "test.k8s.local" + c := &RollingUpdateCluster{ Cloud: mockcloud, MasterInterval: 1 * time.Millisecond, @@ -455,7 +464,7 @@ func TestRollingUpdateNoneNeedUpdateWithForce(t *testing.T) { }, } - err := c.RollingUpdate(groups, &kopsapi.InstanceGroupList{}) + err := c.RollingUpdate(groups, cluster, &kopsapi.InstanceGroupList{}) if err != nil { t.Errorf("Error on rolling update: %v", err) } @@ -488,7 +497,7 @@ func TestRollingUpdateEmptyGroup(t *testing.T) { asgGroups, _ := cloud.Autoscaling().DescribeAutoScalingGroups(&autoscaling.DescribeAutoScalingGroupsInput{}) groups := make(map[string]*cloudinstances.CloudInstanceGroup) - err := c.RollingUpdate(groups, &kopsapi.InstanceGroupList{}) + err := c.RollingUpdate(groups, &kopsapi.Cluster{}, &kopsapi.InstanceGroupList{}) if err != nil { t.Errorf("Error on rolling update: %v", err) } @@ -536,6 +545,9 @@ func TestRollingUpdateUnknownRole(t *testing.T) { mockcloud := awsup.BuildMockAWSCloud("us-east-1", "abc") mockcloud.MockAutoscaling = &mockautoscaling.MockAutoscaling{} + cluster := &kopsapi.Cluster{} + cluster.Name = "test.k8s.local" + c := &RollingUpdateCluster{ Cloud: mockcloud, MasterInterval: 1 * time.Millisecond, @@ -626,7 +638,7 @@ func TestRollingUpdateUnknownRole(t *testing.T) { }, } - err := c.RollingUpdate(groups, &kopsapi.InstanceGroupList{}) + err := c.RollingUpdate(groups, cluster, &kopsapi.InstanceGroupList{}) if err == nil { t.Errorf("Error expected on rolling update: %v", err) } diff --git a/pkg/validation/validate_cluster.go b/pkg/validation/validate_cluster.go index ef018fb4aa..b67b6387d1 100644 --- a/pkg/validation/validate_cluster.go +++ b/pkg/validation/validate_cluster.go @@ -93,7 +93,9 @@ func HasPlaceHolderIP(clusterName string) (bool, error) { } // ValidateCluster validate a k8s cluster with a provided instance group list -func ValidateCluster(clusterName string, instanceGroupList *kops.InstanceGroupList, clusterKubernetesClient kubernetes.Interface) (*ValidationCluster, error) { +func ValidateCluster(cluster *kops.Cluster, instanceGroupList *kops.InstanceGroupList, clusterKubernetesClient kubernetes.Interface) (*ValidationCluster, error) { + clusterName := cluster.Name + var instanceGroups []*kops.InstanceGroup for i := range instanceGroupList.Items {