mirror of https://github.com/kubernetes/kops.git
Merge pull request #4733 from justinsb/int_validation_fix_2
Validation: clean up unused node status functions
This commit is contained in:
k8s-ci-robot
GitHub
commit
963eb0d512
|
|
@ -232,7 +232,7 @@ func validateClusterOutputTable(validationCluster *validation.ValidationCluster,
|
|||
})
|
||||
|
||||
nodeTable.AddColumn("READY", func(n v1.Node) v1.ConditionStatus {
|
||||
return validation.GetNodeConditionStatus(&n)
|
||||
return validation.GetNodeReadyStatus(&n)
|
||||
})
|
||||
|
||||
nodeTable.AddColumn("ROLE", func(n v1.Node) string {
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
|
|||
go_library(
|
||||
name = "go_default_library",
|
||||
srcs = [
|
||||
"node_api_adapter.go",
|
||||
"node_conditions.go",
|
||||
"validate_cluster.go",
|
||||
],
|
||||
importpath = "k8s.io/kops/pkg/validation",
|
||||
|
|
@ -14,7 +14,6 @@ go_library(
|
|||
"//vendor/github.com/golang/glog:go_default_library",
|
||||
"//vendor/k8s.io/api/core/v1:go_default_library",
|
||||
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
|
||||
"//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
|
||||
"//vendor/k8s.io/client-go/kubernetes:go_default_library",
|
||||
"//vendor/k8s.io/client-go/tools/clientcmd:go_default_library",
|
||||
],
|
||||
|
|
@ -23,7 +22,6 @@ go_library(
|
|||
go_test(
|
||||
name = "go_default_test",
|
||||
srcs = [
|
||||
"node_api_adapter_test.go",
|
||||
"validate_cluster_test.go",
|
||||
],
|
||||
embed = [":go_default_library"],
|
||||
|
|
|
|||
|
|
@ -1,249 +0,0 @@
|
|||
/*
|
||||
Copyright 2016 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package validation
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/golang/glog"
|
||||
"k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/util/wait"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
)
|
||||
|
||||
const (
|
||||
// How often to Poll pods, nodes and claims.
|
||||
Poll = 2 * time.Second
|
||||
|
||||
// How long to try single API calls (like 'get' or 'list'). Used to prevent
|
||||
// transient failures
|
||||
// TODO: client should not apply this timeout to Watch calls. Increased from 30s until that is fixed.
|
||||
SingleCallTimeout = 5 * time.Minute
|
||||
)
|
||||
|
||||
// TODO: Rename to NodeValidator; probably just convert to utility functions
|
||||
// NodeAPIAdapter used to retrieve information about Nodes in K8s
|
||||
// TODO: should we pool the api client connection? My initial thought is no.
|
||||
type NodeAPIAdapter struct {
|
||||
// K8s API client this sucker talks to K8s directly - not kubectl, hard api call
|
||||
client kubernetes.Interface
|
||||
|
||||
//TODO: convert to arg on WaitForNodeToBe
|
||||
// K8s timeout on method call
|
||||
timeout time.Duration
|
||||
}
|
||||
|
||||
func NewNodeAPIAdapter(client kubernetes.Interface, timeout time.Duration) (*NodeAPIAdapter, error) {
|
||||
if client == nil {
|
||||
return nil, fmt.Errorf("client not provided")
|
||||
}
|
||||
return &NodeAPIAdapter{
|
||||
client: client,
|
||||
timeout: timeout,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// GetAllNodes is a access to get all nodes from a cluster api
|
||||
func (nodeAA *NodeAPIAdapter) GetAllNodes() (nodes *v1.NodeList, err error) {
|
||||
opts := metav1.ListOptions{}
|
||||
nodes, err = nodeAA.client.CoreV1().Nodes().List(opts)
|
||||
if err != nil {
|
||||
glog.V(4).Infof("getting nodes failed for node %v", err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return nodes, nil
|
||||
}
|
||||
|
||||
// GetReadySchedulableNodes addresses the common use case of getting nodes you can do work on.
|
||||
// 1) Needs to be schedulable.
|
||||
// 2) Needs to be ready.
|
||||
// If EITHER 1 or 2 is not true, most tests will want to ignore the node entirely.
|
||||
func (nodeAA *NodeAPIAdapter) GetReadySchedulableNodes() ([]*v1.Node, error) {
|
||||
nodeList, err := nodeAA.waitListSchedulableNodes()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error from listing schedulable nodes: %v", err)
|
||||
}
|
||||
|
||||
// previous tests may have cause failures of some nodes. Let's skip
|
||||
// 'Not Ready' nodes, just in case (there is no need to fail the test).
|
||||
filtered := FilterNodes(nodeList, isNodeSchedulable)
|
||||
return filtered, err
|
||||
|
||||
}
|
||||
|
||||
// WaitForNodeToBeReady returns whether node name is ready within timeout.
|
||||
func (nodeAA *NodeAPIAdapter) WaitForNodeToBeReady(nodeName string) (bool, error) {
|
||||
return nodeAA.WaitForNodeToBe(nodeName, v1.NodeReady, v1.ConditionTrue)
|
||||
}
|
||||
|
||||
// WaitForNodeToBeNotReady returns whether node is not ready (i.e. the
|
||||
// readiness condition is anything but ready, e.g false or unknown) within
|
||||
// timeout.
|
||||
func (nodeAA *NodeAPIAdapter) WaitForNodeToBeNotReady(nodeName string) (bool, error) {
|
||||
return nodeAA.WaitForNodeToBe(nodeName, v1.NodeReady, v1.ConditionFalse, v1.ConditionUnknown)
|
||||
}
|
||||
|
||||
// WaitForNodeToBe returns whether the names node condition state matches one of the expected values,
|
||||
// within timeout.
|
||||
func (nodeAA *NodeAPIAdapter) WaitForNodeToBe(nodeName string, conditionType v1.NodeConditionType, expected ...v1.ConditionStatus) (bool, error) {
|
||||
if nodeName == "" {
|
||||
return false, fmt.Errorf("nodeName was empty")
|
||||
}
|
||||
|
||||
glog.V(4).Infof("Waiting up to %v for node %s condition %s to be %v", nodeAA.timeout, nodeName, conditionType, expected)
|
||||
|
||||
var cond *v1.NodeCondition
|
||||
err := wait.PollImmediate(Poll, nodeAA.timeout, func() (bool, error) {
|
||||
node, err := nodeAA.client.CoreV1().Nodes().Get(nodeName, metav1.GetOptions{})
|
||||
// FIXME this is not erroring on 500's for instance. We will keep looping
|
||||
if err != nil {
|
||||
// TODO: Check if e.g. NotFound
|
||||
glog.V(4).Infof("Couldn't get node %s: %v", nodeName, err)
|
||||
return false, nil
|
||||
}
|
||||
cond = findNodeCondition(node, conditionType)
|
||||
if cond == nil {
|
||||
return false, nil
|
||||
}
|
||||
return conditionMatchesExpected(cond, expected...), nil
|
||||
})
|
||||
if err != nil {
|
||||
if err == wait.ErrWaitTimeout {
|
||||
glog.V(4).Infof("Node %s didn't reach desired %s condition status (%v) within %v. Actual=%v", nodeName, conditionType, expected, nodeAA.timeout, cond)
|
||||
return false, nil
|
||||
}
|
||||
// TODO: Should return error
|
||||
return false, nil
|
||||
} else {
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
|
||||
// IsNodeConditionUnset check that node condition is not set
|
||||
func isNodeConditionUnset(node *v1.Node, conditionType v1.NodeConditionType) bool {
|
||||
cond := findNodeCondition(node, conditionType)
|
||||
return cond == nil
|
||||
}
|
||||
|
||||
func FilterNodes(nodeList *v1.NodeList, fn func(node *v1.Node) bool) []*v1.Node {
|
||||
var matches []*v1.Node
|
||||
for i := range nodeList.Items {
|
||||
node := &nodeList.Items[i]
|
||||
if fn(node) {
|
||||
matches = append(matches, node)
|
||||
}
|
||||
}
|
||||
return matches
|
||||
}
|
||||
|
||||
// waitListSchedulableNodes is a wrapper around listing nodes supporting retries.
|
||||
func (nodeAA *NodeAPIAdapter) waitListSchedulableNodes() (*v1.NodeList, error) {
|
||||
var nodeList *v1.NodeList
|
||||
err := wait.PollImmediate(Poll, SingleCallTimeout, func() (bool, error) {
|
||||
var err error
|
||||
nodeList, err = nodeAA.client.CoreV1().Nodes().List(metav1.ListOptions{FieldSelector: "spec.unschedulable=false"})
|
||||
if err != nil {
|
||||
// error logging TODO
|
||||
return false, err
|
||||
}
|
||||
return err == nil, nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
// TODO logging
|
||||
return nil, err
|
||||
}
|
||||
return nodeList, err
|
||||
}
|
||||
|
||||
func findNodeCondition(node *v1.Node, conditionType v1.NodeConditionType) *v1.NodeCondition {
|
||||
for i := range node.Status.Conditions {
|
||||
cond := &node.Status.Conditions[i]
|
||||
if cond.Type == conditionType {
|
||||
return cond
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func conditionMatchesExpected(cond *v1.NodeCondition, expected ...v1.ConditionStatus) bool {
|
||||
for _, e := range expected {
|
||||
if cond.Status == e {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
func isNodeConditionSetAsExpected(node *v1.Node, conditionType v1.NodeConditionType, expected ...v1.ConditionStatus) bool {
|
||||
cond := findNodeCondition(node, conditionType)
|
||||
if cond == nil {
|
||||
glog.V(4).Infof("Couldn't find condition %v on node %v", conditionType, node.Name)
|
||||
return false
|
||||
}
|
||||
|
||||
if conditionMatchesExpected(cond, expected...) {
|
||||
return true
|
||||
}
|
||||
|
||||
glog.V(4).Infof(
|
||||
"Condition %s of node %s is %v instead of %v. Reason: %v, message: %v",
|
||||
conditionType, node.Name, cond.Status, expected, cond.Reason, cond.Message)
|
||||
return false
|
||||
}
|
||||
|
||||
// Node is schedulable if:
|
||||
// 1) doesn't have "unschedulable" field set
|
||||
// 2) it's Ready condition is set to true
|
||||
// 3) doesn't have NetworkUnavailable condition set to true
|
||||
func isNodeSchedulable(node *v1.Node) bool {
|
||||
nodeReady := isNodeConditionSetAsExpected(node, v1.NodeReady, v1.ConditionTrue)
|
||||
|
||||
// TODO: Combine
|
||||
networkUnavailable := isNodeConditionUnset(node, v1.NodeNetworkUnavailable)
|
||||
networkUnavailableSilent := isNodeConditionSetAsExpected(node, v1.NodeNetworkUnavailable, v1.ConditionFalse, v1.ConditionUnknown)
|
||||
|
||||
networkReady := networkUnavailable || networkUnavailableSilent
|
||||
|
||||
return !node.Spec.Unschedulable && nodeReady && networkReady
|
||||
}
|
||||
|
||||
// Get The Status of a Node
|
||||
func GetNodeConditionStatus(node *v1.Node) v1.ConditionStatus {
|
||||
cond := findNodeCondition(node, v1.NodeReady)
|
||||
if cond != nil {
|
||||
return cond.Status
|
||||
}
|
||||
return v1.ConditionUnknown
|
||||
}
|
||||
|
||||
// Node is ready if:
|
||||
// 1) its Ready condition is set to true
|
||||
// 2) doesn't have NetworkUnavailable condition set to true
|
||||
func IsNodeOrMasterReady(node *v1.Node) bool {
|
||||
nodeReady := isNodeConditionSetAsExpected(node, v1.NodeReady, v1.ConditionTrue)
|
||||
|
||||
// TODO: Combine
|
||||
networkUnavailable := isNodeConditionUnset(node, v1.NodeNetworkUnavailable)
|
||||
networkUnavailableSilent := isNodeConditionSetAsExpected(node, v1.NodeNetworkUnavailable, v1.ConditionFalse, v1.ConditionUnknown)
|
||||
|
||||
networkReady := networkUnavailable || networkUnavailableSilent
|
||||
|
||||
return nodeReady && networkReady
|
||||
}
|
||||
|
|
@ -1,82 +0,0 @@
|
|||
/*
|
||||
Copyright 2016 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package validation
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/client-go/kubernetes/fake"
|
||||
)
|
||||
|
||||
//func TestBuildNodeAPIAdapter(t *testing.T) {
|
||||
//
|
||||
//}
|
||||
//
|
||||
//func TestGetReadySchedulableNodes(t *testing.T) {
|
||||
//
|
||||
//}
|
||||
|
||||
func TestWaitForNodeToBeReady(t *testing.T) {
|
||||
conditions := []v1.NodeCondition{{Type: "Ready", Status: "True"}}
|
||||
nodeName := "node-foo"
|
||||
nodeAA := setupNodeAA(t, conditions, nodeName)
|
||||
|
||||
test, err := nodeAA.WaitForNodeToBeReady(nodeName)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if test != true {
|
||||
t.Fatalf("unexpected error WaitForNodeToBeReady Failed: %v", test)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWaitForNodeToBeNotReady(t *testing.T) {
|
||||
conditions := []v1.NodeCondition{{Type: "Ready", Status: "False"}}
|
||||
nodeName := "node-foo"
|
||||
nodeAA := setupNodeAA(t, conditions, nodeName)
|
||||
|
||||
test, err := nodeAA.WaitForNodeToBeNotReady(nodeName)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if test != true {
|
||||
t.Fatalf("unexpected error WaitForNodeToBeNotReady Failed: %v", test)
|
||||
}
|
||||
}
|
||||
|
||||
//func TestIsNodeConditionUnset(t *testing.T) {
|
||||
//
|
||||
//}
|
||||
|
||||
func setupNodeAA(t *testing.T, conditions []v1.NodeCondition, nodeName string) *NodeAPIAdapter {
|
||||
node := &v1.Node{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: nodeName},
|
||||
Spec: v1.NodeSpec{Unschedulable: false},
|
||||
Status: v1.NodeStatus{Conditions: conditions},
|
||||
}
|
||||
|
||||
c := fake.NewSimpleClientset(node)
|
||||
//c.Validate(t, response, err)
|
||||
nodeAA, err := NewNodeAPIAdapter(c, time.Duration(10)*time.Millisecond)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error building NodeAPIAdapter: %v", err)
|
||||
}
|
||||
return nodeAA
|
||||
}
|
||||
|
|
@ -0,0 +1,68 @@
|
|||
/*
|
||||
Copyright 2016 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package validation
|
||||
|
||||
import (
|
||||
"github.com/golang/glog"
|
||||
"k8s.io/api/core/v1"
|
||||
)
|
||||
|
||||
func GetNodeReadyStatus(node *v1.Node) v1.ConditionStatus {
|
||||
cond := findNodeCondition(node, v1.NodeReady)
|
||||
if cond != nil {
|
||||
return cond.Status
|
||||
}
|
||||
return v1.ConditionUnknown
|
||||
}
|
||||
|
||||
func findNodeCondition(node *v1.Node, conditionType v1.NodeConditionType) *v1.NodeCondition {
|
||||
for i := range node.Status.Conditions {
|
||||
cond := &node.Status.Conditions[i]
|
||||
if cond.Type == conditionType {
|
||||
return cond
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// isNodeReady returns if a Node is considered ready.
|
||||
// It is considered ready if:
|
||||
// 1) its Ready condition is set to true
|
||||
// 2) doesn't have NetworkUnavailable condition set to true
|
||||
func isNodeReady(node *v1.Node) bool {
|
||||
nodeReadyCondition := findNodeCondition(node, v1.NodeReady)
|
||||
if nodeReadyCondition == nil {
|
||||
glog.Warningf("v1.NodeReady condition not set on node %s", node.Name)
|
||||
return false
|
||||
}
|
||||
if nodeReadyCondition.Status != v1.ConditionTrue {
|
||||
glog.V(4).Infof("node %q not ready: %v", node.Name, nodeReadyCondition)
|
||||
return false
|
||||
}
|
||||
|
||||
networkUnavailableCondition := findNodeCondition(node, v1.NodeNetworkUnavailable)
|
||||
if networkUnavailableCondition != nil {
|
||||
if networkUnavailableCondition.Status != v1.ConditionFalse && networkUnavailableCondition.Status != v1.ConditionUnknown {
|
||||
glog.V(4).Infof("node %q not ready: %v", node.Name, networkUnavailableCondition)
|
||||
return false
|
||||
}
|
||||
} else {
|
||||
glog.V(4).Infof("v1.NodeNetworkUnavailable condition not set on node %s", node.Name)
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
|
@ -19,7 +19,6 @@ package validation
|
|||
import (
|
||||
"fmt"
|
||||
"net/url"
|
||||
"time"
|
||||
|
||||
"net"
|
||||
|
||||
|
|
@ -105,27 +104,19 @@ func ValidateCluster(clusterName string, instanceGroupList *kops.InstanceGroupLi
|
|||
return nil, fmt.Errorf("no InstanceGroup objects found")
|
||||
}
|
||||
|
||||
timeout, err := time.ParseDuration("10s")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot set timeout %q: %v", clusterName, err)
|
||||
}
|
||||
|
||||
nodeAA, err := NewNodeAPIAdapter(clusterKubernetesClient, timeout)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error building node adapter for %q: %v", clusterName, err)
|
||||
}
|
||||
|
||||
validationCluster := &ValidationCluster{
|
||||
ClusterName: clusterName,
|
||||
ErrorMessage: ClusterValidationPassed,
|
||||
InstanceGroups: instanceGroups,
|
||||
}
|
||||
|
||||
validationCluster.NodeList, err = nodeAA.GetAllNodes()
|
||||
nodes, err := clusterKubernetesClient.CoreV1().Nodes().List(metav1.ListOptions{})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot get nodes for %q: %v", clusterName, err)
|
||||
return nil, fmt.Errorf("error querying nodes: %v", err)
|
||||
}
|
||||
|
||||
validationCluster.NodeList = nodes
|
||||
|
||||
validationCluster.ComponentFailures, err = collectComponentFailures(clusterKubernetesClient)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot get component status for %q: %v", clusterName, err)
|
||||
|
|
@ -192,10 +183,10 @@ func validateTheNodes(clusterName string, validationCluster *ValidationCluster)
|
|||
Zone: node.ObjectMeta.Labels["failure-domain.beta.kubernetes.io/zone"],
|
||||
Hostname: node.ObjectMeta.Labels["kubernetes.io/hostname"],
|
||||
Role: role,
|
||||
Status: GetNodeConditionStatus(node),
|
||||
Status: GetNodeReadyStatus(node),
|
||||
}
|
||||
|
||||
ready := IsNodeOrMasterReady(node)
|
||||
ready := isNodeReady(node)
|
||||
|
||||
// TODO: Use instance group role instead...
|
||||
if n.Role == "master" {
|
||||
|
|
|
|||
Loading…
Reference in New Issue