From 6a6bd7d8a025b129346fe9de3f7c723ec0951e70 Mon Sep 17 00:00:00 2001
From: Justin SB <justinsb@google.com>
Date: Fri, 2 Aug 2019 09:19:56 -0400
Subject: [PATCH] Add --wait argument to kops validate

With this argument, kops validate will poll until the timeout expires,
waiting for readiness.  On readiness or on timer expiration, it exits
as if wait was not present.
---
 cmd/kops/validate_cluster.go      | 84 +++++++++++++++++++------------
 docs/cli/kops_validate_cluster.md |  1 +
 2 files changed, 53 insertions(+), 32 deletions(-)

diff --git a/cmd/kops/validate_cluster.go b/cmd/kops/validate_cluster.go
index b353df8153..9e08c108fe 100644
--- a/cmd/kops/validate_cluster.go
+++ b/cmd/kops/validate_cluster.go
@@ -23,6 +23,7 @@ import (
 	"os"
 	"runtime"
 	"strings"
+	"time"
 
 	"github.com/ghodss/yaml"
 	"github.com/spf13/cobra"
@@ -46,6 +47,7 @@ func init() {
 
 type ValidateClusterOptions struct {
 	output string
+	wait   time.Duration
 }
 
 func (o *ValidateClusterOptions) InitDefaults() {
@@ -75,6 +77,7 @@ func NewCmdValidateCluster(f *util.Factory, out io.Writer) *cobra.Command {
 	}
 
 	cmd.Flags().StringVarP(&options.output, "output", "o", options.output, "Output format. One of json|yaml|table.")
+	cmd.Flags().DurationVar(&options.wait, "wait", options.wait, "If set, will wait for cluster to be ready")
 
 	return cmd
 }
@@ -128,40 +131,57 @@ func RunValidateCluster(f *util.Factory, cmd *cobra.Command, args []string, out
 		return nil, fmt.Errorf("Cannot build kubernetes api client for %q: %v", contextName, err)
 	}
 
-	result, err := validation.ValidateCluster(cluster, list, k8sClient)
-	if err != nil {
-		return nil, fmt.Errorf("unexpected error during validation: %v", err)
+	timeout := time.Now().Add(options.wait)
+	pollInterval := 10 * time.Second
+
+	for {
+		result, err := validation.ValidateCluster(cluster, list, k8sClient)
+		if err != nil {
+			if time.Now().After(timeout) {
+				return nil, fmt.Errorf("unexpected error during validation: %v", err)
+			} else {
+				klog.Warningf("(will retry): unexpected error during validation: %v", err)
+				time.Sleep(pollInterval)
+				continue
+			}
+		}
+
+		switch options.output {
+		case OutputTable:
+			if err := validateClusterOutputTable(result, cluster, instanceGroups, out); err != nil {
+				return nil, err
+			}
+
+		case OutputYaml:
+			y, err := yaml.Marshal(result)
+			if err != nil {
+				return nil, fmt.Errorf("unable to marshal YAML: %v", err)
+			}
+			if _, err := out.Write(y); err != nil {
+				return nil, fmt.Errorf("error writing to output: %v", err)
+			}
+
+		case OutputJSON:
+			j, err := json.Marshal(result)
+			if err != nil {
+				return nil, fmt.Errorf("unable to marshal JSON: %v", err)
+			}
+			if _, err := out.Write(j); err != nil {
+				return nil, fmt.Errorf("error writing to output: %v", err)
+			}
+
+		default:
+			return nil, fmt.Errorf("Unknown output format: %q", options.output)
+		}
+
+		if options.wait == 0 || len(result.Failures) == 0 {
+			return result, nil
+		}
+
+		klog.Warningf("(will retry): cluster not yet healthy")
+		time.Sleep(pollInterval)
 	}
 
-	switch options.output {
-	case OutputTable:
-		if err := validateClusterOutputTable(result, cluster, instanceGroups, out); err != nil {
-			return nil, err
-		}
-
-	case OutputYaml:
-		y, err := yaml.Marshal(result)
-		if err != nil {
-			return nil, fmt.Errorf("unable to marshal YAML: %v", err)
-		}
-		if _, err := out.Write(y); err != nil {
-			return nil, fmt.Errorf("error writing to output: %v", err)
-		}
-
-	case OutputJSON:
-		j, err := json.Marshal(result)
-		if err != nil {
-			return nil, fmt.Errorf("unable to marshal JSON: %v", err)
-		}
-		if _, err := out.Write(j); err != nil {
-			return nil, fmt.Errorf("error writing to output: %v", err)
-		}
-
-	default:
-		return nil, fmt.Errorf("Unknown output format: %q", options.output)
-	}
-
-	return result, nil
 }
 
 func validateClusterOutputTable(result *validation.ValidationCluster, cluster *api.Cluster, instanceGroups []api.InstanceGroup, out io.Writer) error {
diff --git a/docs/cli/kops_validate_cluster.md b/docs/cli/kops_validate_cluster.md
index d061e8858e..f0e5b4527a 100644
--- a/docs/cli/kops_validate_cluster.md
+++ b/docs/cli/kops_validate_cluster.md
@@ -32,6 +32,7 @@ kops validate cluster [flags]
 ```
   -h, --help            help for cluster
   -o, --output string   Output format. One of json|yaml|table. (default "table")
+      --wait duration   If set, will wait for cluster to be ready
 ```
 
 ### Options inherited from parent commands