adding component and kube-system validation

cmd/kops/validate_cluster.go

@@ -165,6 +165,24 @@ func RunValidateCluster(f *util.Factory, cmd *cobra.Command, args []string, out
 		return fmt.Errorf("cannot render nodes for %q: %v", cluster.ObjectMeta.Name, err)
 	}
 
+	if len(validationCluster.ComponentFailures) != 0 {
+		fmt.Fprintln(out, "\nComponent Failures")
+		err = t.Render(validationCluster.ComponentFailures, out, "NAME")
+
+		if err != nil {
+			return fmt.Errorf("cannot render components for %q: %v", cluster.ObjectMeta.Name, err)
+		}
+	}
+
+	if len(validationCluster.PodFailures) != 0 {
+		fmt.Fprintln(out, "\nPod Failures in kube-system")
+		err = t.Render(validationCluster.PodFailures, out, "NAME")
+
+		if err != nil {
+			return fmt.Errorf("cannot render pods for %q: %v", cluster.ObjectMeta.Name, err)
+		}
+	}
+
 	if validationFailed == nil {
 		fmt.Fprintf(out, "\nYour cluster %s is ready\n", cluster.ObjectMeta.Name)
 		return nil
@@ -174,6 +192,6 @@ func RunValidateCluster(f *util.Factory, cmd *cobra.Command, args []string, out
 		fmt.Fprint(out, "\nValidation Failed\n")
 		fmt.Fprintf(out, "Ready Master(s) %d out of %d.\n", len(validationCluster.MastersReadyArray), validationCluster.MastersCount)
 		fmt.Fprintf(out, "Ready Node(s) %d out of %d.\n", len(validationCluster.NodesReadyArray), validationCluster.NodesCount)
-		return fmt.Errorf("Your cluster %s is NOT ready.\n", cluster.ObjectMeta.Name)
+		return validationFailed
 	}
 }

pkg/validation/validate_cluster.go

@@ -39,6 +39,9 @@ type ValidationCluster struct {
 	NodesCount         int               `json:"nodesCount,omitempty"`
 
 	NodeList *v1.NodeList `json:"nodeList,omitempty"`
+
+	ComponentFailures []string `json:"componentFailures,omitempty"`
+	PodFailures       []string `json:"podFailures,omitempty"`
 }
 
 // A K8s node to be validated
@@ -65,12 +68,12 @@ func ValidateCluster(clusterName string, instanceGroupList *kops.InstanceGroupLi
 	}
 
 	if len(instanceGroups) == 0 {
-		return validationCluster, fmt.Errorf("No InstanceGroup objects found")
+		return validationCluster, fmt.Errorf("no InstanceGroup objects found")
 	}
 
 	timeout, err := time.ParseDuration("10s")
 	if err != nil {
-		return nil, fmt.Errorf("Cannot set timeout %q: %v", clusterName, err)
+		return nil, fmt.Errorf("cannot set timeout %q: %v", clusterName, err)
 	}
 
 	nodeAA, err := NewNodeAPIAdapter(clusterKubernetesClient, timeout)
@@ -80,7 +83,17 @@ func ValidateCluster(clusterName string, instanceGroupList *kops.InstanceGroupLi
 
 	validationCluster.NodeList, err = nodeAA.GetAllNodes()
 	if err != nil {
-		return nil, fmt.Errorf("Cannot reach API server: %v", err)
+		return nil, fmt.Errorf("cannot get nodes for %q: %v", clusterName, err)
+	}
+
+	validationCluster.ComponentFailures, err = collectComponentFailures(clusterKubernetesClient)
+	if err != nil {
+		return nil, fmt.Errorf("cannot get component status for %q: %v", clusterName, err)
+	}
+
+	validationCluster.PodFailures, err = collectPodFailures(clusterKubernetesClient)
+	if err != nil {
+		return nil, fmt.Errorf("cannot get pod health for %q: %v", clusterName, err)
 	}
 
 	return validateTheNodes(clusterName, validationCluster)
@@ -96,6 +109,34 @@ func getRoleNode(node *v1.Node) string {
 	return role
 }
 
+func collectComponentFailures(client k8s_clientset.Interface) (failures []string, err error) {
+	componentList, err := client.CoreV1().ComponentStatuses().List(v1.ListOptions{})
+	if err == nil {
+		for _, component := range componentList.Items {
+			for _, condition := range component.Conditions {
+				if condition.Status != v1.ConditionTrue {
+					failures = append(failures, component.Name)
+				}
+			}
+		}
+	}
+	return
+}
+
+func collectPodFailures(client k8s_clientset.Interface) (failures []string, err error) {
+	pods, err := client.CoreV1().Pods("kube-system").List(v1.ListOptions{})
+	if err == nil {
+		for _, pod := range pods.Items {
+			for _, status := range pod.Status.ContainerStatuses {
+				if !status.Ready {
+					failures = append(failures, pod.Name)
+				}
+			}
+		}
+	}
+	return
+}
+
 func validateTheNodes(clusterName string, validationCluster *ValidationCluster) (*ValidationCluster, error) {
 	nodes := validationCluster.NodeList
 
@@ -144,10 +185,21 @@ func validateTheNodes(clusterName string, validationCluster *ValidationCluster)
 		validationCluster.NodesReady = false
 	}
 
-	if validationCluster.MastersReady && validationCluster.NodesReady {
-		return validationCluster, nil
-	} else {
-		// TODO: This isn't an error...
-		return validationCluster, fmt.Errorf("Your cluster is NOT ready %s", clusterName)
+	if !validationCluster.MastersReady {
+		return validationCluster, fmt.Errorf("your masters are NOT ready %s", clusterName)
 	}
+
+	if !validationCluster.NodesReady {
+		return validationCluster, fmt.Errorf("your nodes are NOT ready %s", clusterName)
+	}
+
+	if len(validationCluster.ComponentFailures) != 0 {
+		return validationCluster, fmt.Errorf("your components are NOT healthy %s", clusterName)
+	}
+
+	if len(validationCluster.PodFailures) != 0 {
+		return validationCluster, fmt.Errorf("your kube-system pods are NOT healthy %s", clusterName)
+	}
+
+	return validationCluster, nil
 }

pkg/validation/validate_cluster_test.go

@@ -57,6 +57,40 @@ func Test_ValidateClusterMasterAndNodeNotReady(t *testing.T) {
 	}
 }
 
+func Test_ValidateClusterComponents(t *testing.T) {
+	nodeList, err := dummyClient("true", "true").Core().Nodes().List(v1.ListOptions{})
+
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+
+	var component = make([]string, 1)
+	validationCluster := &ValidationCluster{NodeList: nodeList, NodesCount: 1, MastersCount: 1, ComponentFailures: component}
+	validationCluster, err = validateTheNodes("foo", validationCluster)
+
+	if err == nil {
+		printDebug(validationCluster)
+		t.Fatalf("unexpected error: %v", err)
+	}
+}
+
+func Test_ValidateClusterPods(t *testing.T) {
+	nodeList, err := dummyClient("true", "true").Core().Nodes().List(v1.ListOptions{})
+
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+
+	var pod = make([]string, 1)
+	validationCluster := &ValidationCluster{NodeList: nodeList, NodesCount: 1, MastersCount: 1, PodFailures: pod}
+	validationCluster, err = validateTheNodes("foo", validationCluster)
+
+	if err == nil {
+		printDebug(validationCluster)
+		t.Fatalf("unexpected error: %v", err)
+	}
+}
+
 func Test_ValidateClusterNodeNotReady(t *testing.T) {
 	nodeList, err := dummyClient("true", "false").Core().Nodes().List(v1.ListOptions{})