mirror of https://github.com/kubernetes/kops.git
Starting work on node lookup and validation
This commit is contained in:
parent
89ac0d24f8
commit
3ffc4e91cb
|
|
@ -17,12 +17,12 @@ limitations under the License.
|
||||||
package kutil
|
package kutil
|
||||||
|
|
||||||
import (
|
import (
|
||||||
client "k8s.io/kubernetes/pkg/client/unversioned"
|
|
||||||
"time"
|
|
||||||
"k8s.io/kubernetes/pkg/api"
|
|
||||||
"github.com/golang/glog"
|
"github.com/golang/glog"
|
||||||
"k8s.io/kubernetes/pkg/util/wait"
|
"k8s.io/kubernetes/pkg/api"
|
||||||
|
client "k8s.io/kubernetes/pkg/client/unversioned"
|
||||||
"k8s.io/kubernetes/pkg/fields"
|
"k8s.io/kubernetes/pkg/fields"
|
||||||
|
"k8s.io/kubernetes/pkg/util/wait"
|
||||||
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
|
|
@ -35,45 +35,65 @@ const (
|
||||||
SingleCallTimeout = 5 * time.Minute
|
SingleCallTimeout = 5 * time.Minute
|
||||||
)
|
)
|
||||||
|
|
||||||
|
type NodeAPIAdapter struct {
|
||||||
|
client *client.Client
|
||||||
|
timeout time.Duration
|
||||||
|
}
|
||||||
|
|
||||||
// GetReadySchedulableNodesOrDie addresses the common use case of getting nodes you can do work on.
|
// GetReadySchedulableNodesOrDie addresses the common use case of getting nodes you can do work on.
|
||||||
// 1) Needs to be schedulable.
|
// 1) Needs to be schedulable.
|
||||||
// 2) Needs to be ready.
|
// 2) Needs to be ready.
|
||||||
// If EITHER 1 or 2 is not true, most tests will want to ignore the node entirely.
|
// If EITHER 1 or 2 is not true, most tests will want to ignore the node entirely.
|
||||||
func GetReadySchedulableNodesOrDie(c *client.Client) (nodes *api.NodeList) {
|
func (nodeAA *NodeAPIAdapter) GetReadySchedulableNodes() (nodes *api.NodeList, err error) {
|
||||||
nodes = waitListSchedulableNodesOrDie(c)
|
|
||||||
|
nodes, err = nodeAA.waitListSchedulableNodes()
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
// previous tests may have cause failures of some nodes. Let's skip
|
// previous tests may have cause failures of some nodes. Let's skip
|
||||||
// 'Not Ready' nodes, just in case (there is no need to fail the test).
|
// 'Not Ready' nodes, just in case (there is no need to fail the test).
|
||||||
FilterNodes(nodes, func(node api.Node) bool {
|
FilterNodes(nodes, func(node api.Node) bool {
|
||||||
return isNodeSchedulable(&node)
|
return isNodeSchedulable(&node), nil
|
||||||
})
|
})
|
||||||
return nodes
|
return nodes, err
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// waitListSchedulableNodesOrDie is a wrapper around listing nodes supporting retries.
|
// WaitForNodeToBeReady returns whether node name is ready within timeout.
|
||||||
func waitListSchedulableNodesOrDie(c *client.Client) *api.NodeList {
|
func (nodeAA *NodeAPIAdapter) WaitForNodeToBeReady(name string) bool {
|
||||||
var nodes *api.NodeList
|
return nodeAA.WaitForNodeToBe(name, api.NodeReady, true)
|
||||||
var err error
|
}
|
||||||
if wait.PollImmediate(Poll, SingleCallTimeout, func() (bool, error) {
|
|
||||||
nodes, err = c.Nodes().List(api.ListOptions{FieldSelector: fields.Set{
|
// WaitForNodeToBeNotReady returns whether node name is not ready (i.e. the
|
||||||
"spec.unschedulable": "false",
|
// readiness condition is anything but ready, e.g false or unknown) within
|
||||||
}.AsSelector()})
|
// timeout.
|
||||||
return err == nil, nil
|
func (nodeAA *NodeAPIAdapter) WaitForNodeToBeNotReady(name string) bool {
|
||||||
}) != nil {
|
return nodeAA.WaitForNodeToBe(name, api.NodeReady, false)
|
||||||
ExpectNoError(err, "Timed out while listing nodes for e2e cluster.")
|
}
|
||||||
|
|
||||||
|
// WaitForNodeToBe returns whether node "name's" condition state matches wantTrue
|
||||||
|
// within timeout. If wantTrue is true, it will ensure the node condition status
|
||||||
|
// is ConditionTrue; if it's false, it ensures the node condition is in any state
|
||||||
|
// other than ConditionTrue (e.g. not true or unknown).
|
||||||
|
func (nodeAA *NodeAPIAdapter) WaitForNodeToBe(name string, conditionType api.NodeConditionType, wantTrue bool) bool {
|
||||||
|
glog.V(4).Infof("Waiting up to %v for node %s condition %s to be %t", nodeAA.timeout, name, conditionType, wantTrue)
|
||||||
|
for start := time.Now(); time.Since(start) < nodeAA.timeout; time.Sleep(Poll) {
|
||||||
|
node, err := nodeAA.client.Nodes().Get(name)
|
||||||
|
|
||||||
|
// FIXME this is not erroring on 500's for instance. We will keep looping
|
||||||
|
if err != nil {
|
||||||
|
glog.V(4).Infof("Couldn't get node %s", name)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if IsNodeConditionSetAsExpected(node, conditionType, wantTrue) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return nodes
|
glog.V(4).Infof("Node %s didn't reach desired %s condition status (%t) within %v", name, conditionType, wantTrue, nodeAA.timeout)
|
||||||
}
|
return false
|
||||||
|
|
||||||
// Node is schedulable if:
|
|
||||||
// 1) doesn't have "unschedulable" field set
|
|
||||||
// 2) it's Ready condition is set to true
|
|
||||||
// 3) doesn't have NetworkUnavailable condition set to true
|
|
||||||
func isNodeSchedulable(node *api.Node) bool {
|
|
||||||
nodeReady := IsNodeConditionSetAsExpected(node, api.NodeReady, true)
|
|
||||||
networkReady := IsNodeConditionUnset(node, api.NodeNetworkUnavailable) ||
|
|
||||||
IsNodeConditionSetAsExpectedSilent(node, api.NodeNetworkUnavailable, false)
|
|
||||||
return !node.Spec.Unschedulable && nodeReady && networkReady
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func IsNodeConditionSetAsExpectedSilent(node *api.Node, conditionType api.NodeConditionType, wantTrue bool) bool {
|
func IsNodeConditionSetAsExpectedSilent(node *api.Node, conditionType api.NodeConditionType, wantTrue bool) bool {
|
||||||
|
|
@ -99,58 +119,27 @@ func FilterNodes(nodeList *api.NodeList, fn func(node api.Node) bool) {
|
||||||
nodeList.Items = l
|
nodeList.Items = l
|
||||||
}
|
}
|
||||||
|
|
||||||
// waitListSchedulableNodesOrDie is a wrapper around listing nodes supporting retries.
|
func IsNodeConditionSetAsExpected(node *api.Node, conditionType api.NodeConditionType, wantTrue bool) bool {
|
||||||
func waitListSchedulableNodesOrDie(c *client.Client) (*api.NodeList, error) {
|
return isNodeConditionSetAsExpected(node, conditionType, wantTrue, false)
|
||||||
var nodes *api.NodeList
|
}
|
||||||
var err error
|
|
||||||
|
// waitListSchedulableNodes is a wrapper around listing nodes supporting retries.
|
||||||
|
func (nodeAA *NodeAPIAdapter) waitListSchedulableNodes() (nodes *api.NodeList, err error) {
|
||||||
|
|
||||||
if wait.PollImmediate(Poll, SingleCallTimeout, func() (bool, error) {
|
if wait.PollImmediate(Poll, SingleCallTimeout, func() (bool, error) {
|
||||||
nodes, err = c.Nodes().List(api.ListOptions{FieldSelector: fields.Set{
|
nodes, err = nodeAA.client.Nodes().List(api.ListOptions{FieldSelector: fields.Set{
|
||||||
"spec.unschedulable": "false",
|
"spec.unschedulable": "false",
|
||||||
}.AsSelector()})
|
}.AsSelector()})
|
||||||
return nodes, err
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return nodes, nil
|
||||||
}) != nil {
|
}) != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
return nodes, nil
|
return nodes, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// WaitForNodeToBeReady returns whether node name is ready within timeout.
|
|
||||||
func WaitForNodeToBeReady(c *client.Client, name string, timeout time.Duration) bool {
|
|
||||||
return WaitForNodeToBe(c, name, api.NodeReady, true, timeout)
|
|
||||||
}
|
|
||||||
|
|
||||||
// WaitForNodeToBeNotReady returns whether node name is not ready (i.e. the
|
|
||||||
// readiness condition is anything but ready, e.g false or unknown) within
|
|
||||||
// timeout.
|
|
||||||
func WaitForNodeToBeNotReady(c *client.Client, name string, timeout time.Duration) bool {
|
|
||||||
return WaitForNodeToBe(c, name, api.NodeReady, false, timeout)
|
|
||||||
}
|
|
||||||
|
|
||||||
// WaitForNodeToBe returns whether node "name's" condition state matches wantTrue
|
|
||||||
// within timeout. If wantTrue is true, it will ensure the node condition status
|
|
||||||
// is ConditionTrue; if it's false, it ensures the node condition is in any state
|
|
||||||
// other than ConditionTrue (e.g. not true or unknown).
|
|
||||||
func WaitForNodeToBe(c *client.Client, name string, conditionType api.NodeConditionType, wantTrue bool, timeout time.Duration) bool {
|
|
||||||
glog.V(4).Infof("Waiting up to %v for node %s condition %s to be %t", timeout, name, conditionType, wantTrue)
|
|
||||||
for start := time.Now(); time.Since(start) < timeout; time.Sleep(Poll) {
|
|
||||||
node, err := c.Nodes().Get(name)
|
|
||||||
if err != nil {
|
|
||||||
glog.V(4).Infof("Couldn't get node %s", name)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
if IsNodeConditionSetAsExpected(node, conditionType, wantTrue) {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
glog.V(4).Infof("Node %s didn't reach desired %s condition status (%t) within %v", name, conditionType, wantTrue, timeout)
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
func IsNodeConditionSetAsExpected(node *api.Node, conditionType api.NodeConditionType, wantTrue bool) bool {
|
|
||||||
return isNodeConditionSetAsExpected(node, conditionType, wantTrue, false)
|
|
||||||
}
|
|
||||||
|
|
||||||
func isNodeConditionSetAsExpected(node *api.Node, conditionType api.NodeConditionType, wantTrue, silent bool) bool {
|
func isNodeConditionSetAsExpected(node *api.Node, conditionType api.NodeConditionType, wantTrue, silent bool) bool {
|
||||||
// Check the node readiness condition (logging all).
|
// Check the node readiness condition (logging all).
|
||||||
for _, cond := range node.Status.Conditions {
|
for _, cond := range node.Status.Conditions {
|
||||||
|
|
@ -173,3 +162,14 @@ func isNodeConditionSetAsExpected(node *api.Node, conditionType api.NodeConditio
|
||||||
}
|
}
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Node is schedulable if:
|
||||||
|
// 1) doesn't have "unschedulable" field set
|
||||||
|
// 2) it's Ready condition is set to true
|
||||||
|
// 3) doesn't have NetworkUnavailable condition set to true
|
||||||
|
func isNodeSchedulable(node *api.Node) bool {
|
||||||
|
nodeReady := IsNodeConditionSetAsExpected(node, api.NodeReady, true)
|
||||||
|
networkReady := IsNodeConditionUnset(node, api.NodeNetworkUnavailable) ||
|
||||||
|
IsNodeConditionSetAsExpectedSilent(node, api.NodeNetworkUnavailable, false)
|
||||||
|
return !node.Spec.Unschedulable && nodeReady && networkReady
|
||||||
|
}
|
||||||
Loading…
Reference in New Issue