working on the start of validate

cluster validation seem to be working.  Need to test more

documentation yo

refactoring to get rid of import cycle not allowed
This commit is contained in:
chrislovecnm 2016-10-20 13:16:42 -06:00
parent 3ffc4e91cb
commit 82cf2c2c0f
7 changed files with 1004 additions and 178 deletions

45
cmd/kops/validate.go Normal file
View File

@ -0,0 +1,45 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package main
import (
"github.com/spf13/cobra"
)
// ValidateCmd represents the get command
type ValidateCmd struct {
output string
cobraCommand *cobra.Command
}
var validateCmd = ValidateCmd{
cobraCommand: &cobra.Command{
Use: "validate",
SuggestFor: []string{"list"},
Short: "validate a kubernetes cluster",
Long: `validate a kubernetes cluster`,
},
}
func init() {
cmd := validateCmd.cobraCommand
rootCommand.AddCommand(cmd)
cmd.PersistentFlags().StringVarP(&validateCmd.output, "output", "o", OutputTable, "output format. One of: table, yaml")
}

View File

@ -0,0 +1,232 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package main
import (
"strings"
"github.com/spf13/cobra"
api "k8s.io/kops/pkg/apis/kops"
"k8s.io/kops/util/pkg/tables"
k8sapi "k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/api/v1"
"errors"
"fmt"
"os"
"time"
"github.com/golang/glog"
)
// not used too much yet :)
type ValidateClusterCmd struct {
FullSpec bool
}
var validateClusterCmd ValidateClusterCmd
// Init darn it
func init() {
cmd := &cobra.Command{
Use: "cluster",
Aliases: []string{"cluster"},
Short: "Validate cluster",
Long: `Validate a kubernetes cluster`,
Run: func(cmd *cobra.Command, args []string) {
err := validateClusterCmd.Run(args)
if err != nil {
exitWithError(err)
}
},
}
validateCmd.cobraCommand.AddCommand(cmd)
//cmd.Flags().BoolVar(&validateClusterCmd.FullSpec, "full", false, "Validate a cluster")
}
// Run a validation
func (c *ValidateClusterCmd) Run(args []string) error {
err := rootCommand.ProcessArgs(args)
if err != nil {
return err
}
cluster, err := rootCommand.Cluster()
if err != nil {
return err
}
clientset, err := rootCommand.Clientset()
if err != nil {
return err
}
nodeAA := &api.NodeAPIAdapter{}
timeout, err := time.ParseDuration("30s")
if err != nil {
return fmt.Errorf("Cannot set timeout %q: %v", cluster.Name, err)
}
nodeAA.BuildNodeAPIAdapter(cluster.Name, timeout, "")
nodes, err := nodeAA.GetAllNodes()
if err != nil {
return fmt.Errorf("Cannot get nodes for %q: %v", cluster.Name, err)
}
list, err := clientset.InstanceGroups(cluster.Name).List(k8sapi.ListOptions{})
if err != nil {
return fmt.Errorf("Cannot get nodes for %q: %v", cluster.Name, err)
}
fmt.Printf("Validating cluster %v\n\n", cluster.Name)
var instancegroups []*api.InstanceGroup
validationCluster := &api.ValidationCluster{}
for i := range list.Items {
ig := &list.Items[i]
instancegroups = append(instancegroups, ig)
if ig.Spec.Role == api.InstanceGroupRoleMaster {
//validationCluster.MastersInstanceGroups = append(validationCluster.MastersInstanceGroups, ig)
validationCluster.MastersCount += *ig.Spec.MinSize
} else {
//validationCluster.NodesInstanceGroups = append(validationCluster.NodesInstanceGroups, ig)
validationCluster.NodesCount += *ig.Spec.MinSize
}
}
if len(instancegroups) == 0 {
return errors.New("No InstanceGroup objects found\n")
}
t := &tables.Table{}
t.AddColumn("NAME", func(c *api.InstanceGroup) string {
return c.Name
})
t.AddColumn("ROLE", func(c *api.InstanceGroup) string {
return string(c.Spec.Role)
})
t.AddColumn("MACHINETYPE", func(c *api.InstanceGroup) string {
return c.Spec.MachineType
})
t.AddColumn("ZONES", func(c *api.InstanceGroup) string {
return strings.Join(c.Spec.Zones, ",")
})
t.AddColumn("MIN", func(c *api.InstanceGroup) string {
return intPointerToString(c.Spec.MinSize)
})
t.AddColumn("MAX", func(c *api.InstanceGroup) string {
return intPointerToString(c.Spec.MaxSize)
})
fmt.Println("INSTANCE GROUPS")
err = t.Render(instancegroups, os.Stdout, "NAME", "ROLE", "MACHINETYPE", "MIN", "MAX", "ZONES")
if err != nil {
return fmt.Errorf("Cannot render nodes for %q: %v", cluster.Name, err)
}
t = &tables.Table{}
t.AddColumn("NAME", func(n v1.Node) string {
return n.Name
})
t.AddColumn("READY", func(n v1.Node) v1.ConditionStatus {
return api.GetNodeConditionStatus(n.Status.Conditions)
})
t.AddColumn("ROLE", func(n v1.Node) string {
role := "node"
if val, ok := n.ObjectMeta.Labels["kubernetes.io/role"]; ok {
role = val
}
return role
})
fmt.Println("\nNODE STATUS")
err = t.Render(nodes.Items, os.Stdout, "NAME", "ROLE", "READY")
if err != nil {
return fmt.Errorf("Cannot render nodes for %q: %v", cluster.Name, err)
}
for _, node := range nodes.Items {
role := "node"
if val, ok := node.ObjectMeta.Labels["kubernetes.io/role"]; ok {
role = val
}
n := &api.ValidationNode{
Zone: node.ObjectMeta.Labels["failure-domain.beta.kubernetes.io/zone"],
Hostname: node.ObjectMeta.Labels["kubernetes.io/hostname"],
Role: role,
Status: api.GetNodeConditionStatus(node.Status.Conditions),
}
if n.Role == "master" {
if n.Status == v1.ConditionTrue {
validationCluster.MastersReady = append(validationCluster.MastersReady, n)
} else {
validationCluster.MastersNotReady = append(validationCluster.MastersNotReady, n)
}
} else if n.Role == "node" {
if n.Status == v1.ConditionTrue {
validationCluster.NodesReady = append(validationCluster.NodesReady, n)
} else {
validationCluster.NodesNotReady = append(validationCluster.NodesNotReady, n)
}
}
}
mastersReady := true
nodesReady := true
if len(validationCluster.MastersNotReady) != 0 || validationCluster.MastersCount !=
len(validationCluster.MastersReady) {
mastersReady = false
}
if len(validationCluster.NodesNotReady) != 0 || validationCluster.NodesCount !=
len(validationCluster.NodesReady) {
nodesReady = false
}
if mastersReady && nodesReady {
fmt.Printf("\nYour cluster %s is ready\n", cluster.Name)
return nil
} else {
// do we need to print which instance group is not ready?
// nodes are going to be a pain
glog.Infof("cluster - masters ready: %v, nodes ready: %v", mastersReady, nodesReady)
glog.Infof("mastersNotReady %v", len(validationCluster.MastersNotReady))
glog.Infof("mastersCount %v, mastersReady %v", validationCluster.MastersCount, len(validationCluster.MastersReady))
glog.Infof("nodesNotReady %v", len(validationCluster.NodesNotReady))
glog.Infof("nodesCount %v, nodesReady %v", validationCluster.NodesCount, len(validationCluster.NodesReady))
return fmt.Errorf("You cluster is NOT ready %s", cluster.Name)
}
}

View File

@ -0,0 +1,372 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package kops
import (
"time"
"fmt"
"github.com/golang/glog"
"github.com/pkg/errors"
"k8s.io/kubernetes/pkg/api/v1"
"k8s.io/kubernetes/pkg/client/clientset_generated/release_1_5"
"k8s.io/kubernetes/pkg/client/unversioned/clientcmd"
"k8s.io/kubernetes/pkg/util/wait"
)
const (
// How often to Poll pods, nodes and claims.
Poll = 2 * time.Second
// How long to try single API calls (like 'get' or 'list'). Used to prevent
// transient failures
// TODO: client should not apply this timeout to Watch calls. Increased from 30s until that is fixed.
SingleCallTimeout = 5 * time.Minute
)
// NodeAPIAdapter used to retrieve information about Nodes in K8s
// TODO: should we pool the api client connection? My initial thought is no.
type NodeAPIAdapter struct {
// K8s API client this sucker talks to K8s directly - not kubectl, hard api call
client interface{}
// K8s timeout on method call
timeout time.Duration
// K8s node name if applicable
nodeName string
}
// Create a NodeAPIAdapter with K8s client based on the current kubectl config
// TODO I do not really like this .... hrmm
func (nodeAA *NodeAPIAdapter) BuildNodeAPIAdapter(clusterName string, timeout time.Duration, nodeName string) (err error) {
config, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
clientcmd.NewDefaultClientConfigLoadingRules(),
&clientcmd.ConfigOverrides{CurrentContext: clusterName}).ClientConfig()
if err != nil {
return fmt.Errorf("cannot load kubecfg settings for %q: %v", clusterName, err)
}
c, err := release_1_5.NewForConfig(config)
if err != nil {
return fmt.Errorf("cannot build kube client for %q: %v", clusterName, err)
}
if err != nil {
return fmt.Errorf("creating client go boom, %v", err)
}
nodeAA.client = c
nodeAA.timeout = timeout
nodeAA.nodeName = nodeName
return nil
}
// GetAllNodes is a access to get all nodes from a cluster api
func (nodeAA *NodeAPIAdapter) GetAllNodes() (nodes *v1.NodeList, err error) {
c, err := nodeAA.getClient()
if err != nil {
glog.V(4).Infof("getClient failed for node %s, %v", nodeAA.nodeName, err)
return nil, err
}
opts := v1.ListOptions{}
nodes, err = c.Nodes().List(opts)
if err != nil {
glog.V(4).Infof("getting nodes failed for node %v", err)
return nil, err
}
return nodes, nil
}
// GetReadySchedulableNodesOrDie addresses the common use case of getting nodes you can do work on.
// 1) Needs to be schedulable.
// 2) Needs to be ready.
// If EITHER 1 or 2 is not true, most tests will want to ignore the node entirely.
func (nodeAA *NodeAPIAdapter) GetReadySchedulableNodes() (nodes *v1.NodeList, err error) {
nodes, err = nodeAA.waitListSchedulableNodes()
if err != nil {
return nil, fmt.Errorf("GetReadySchedulableNodes go boom %v", err)
}
// previous tests may have cause failures of some nodes. Let's skip
// 'Not Ready' nodes, just in case (there is no need to fail the test).
FilterNodes(nodes, func(node v1.Node) (bool, error) {
return isNodeSchedulable(&node)
})
return nodes, err
}
// WaitForNodeToBeReady returns whether node name is ready within timeout.
func (nodeAA *NodeAPIAdapter) WaitForNodeToBeReady() (bool, error) {
return nodeAA.WaitForNodeToBe(v1.NodeReady, true)
}
// WaitForNodeToBeNotReady returns whether node name is not ready (i.e. the
// readiness condition is anything but ready, e.g false or unknown) within
// timeout.
func (nodeAA *NodeAPIAdapter) WaitForNodeToBeNotReady() (bool, error) {
return nodeAA.WaitForNodeToBe(v1.NodeReady, false)
}
// WaitForNodeToBe returns whether node "name's" condition state matches wantTrue
// within timeout. If wantTrue is true, it will ensure the node condition status
// is ConditionTrue; if it's false, it ensures the node condition is in any state
// other than ConditionTrue (e.g. not true or unknown).
func (nodeAA *NodeAPIAdapter) WaitForNodeToBe(conditionType v1.NodeConditionType, wantTrue bool) (bool, error) {
if err := nodeAA.isNodeNameDefined(); err != nil {
return false, fmt.Errorf("isNodeNameDefined failed for node %s, %v", nodeAA.nodeName, err)
}
if err := nodeAA.isClientDefined(); err != nil {
return false, fmt.Errorf("isClientDefined failed for node %s, %v", nodeAA.nodeName, err)
}
glog.V(4).Infof("Waiting up to %v for node %s condition %s to be %t", nodeAA.timeout, nodeAA.nodeName, conditionType, wantTrue)
for start := time.Now(); time.Since(start) < nodeAA.timeout; time.Sleep(Poll) {
c, err := nodeAA.getClient()
if err != nil {
glog.V(4).Infof("getClient failed for node %s, %v", nodeAA.nodeName, err)
return false, err
}
node, err := c.Nodes().Get(nodeAA.nodeName)
// FIXME this is not erroring on 500's for instance. We will keep looping
if err != nil {
glog.V(4).Infof("Couldn't get node %s", nodeAA.nodeName)
continue
}
var iSet bool
iSet, err = IsNodeConditionSetAsExpected(node, conditionType, wantTrue)
if err != nil {
glog.V(4).Infof("IsNodeConditionSetAsExpected failed for node %s, %v", nodeAA.nodeName, err)
return false, err
}
if iSet {
return true, nil
}
}
glog.V(4).Infof("Node %s didn't reach desired %s condition status (%t) within %v", nodeAA.nodeName, conditionType, wantTrue, nodeAA.timeout)
return false, nil
}
// IsNodeConditionSetAsExpectedSilent node conidtion is
func IsNodeConditionSetAsExpectedSilent(node *v1.Node, conditionType v1.NodeConditionType, wantTrue bool) (bool, error) {
return isNodeConditionSetAsExpected(node, conditionType, wantTrue, true)
}
// IsNodeConditionUnset check that node condition is not set
func IsNodeConditionUnset(node *v1.Node, conditionType v1.NodeConditionType) (bool, error) {
if err := isNodeStatusDefined(node); err != nil {
return false, err
}
for _, cond := range node.Status.Conditions {
if cond.Type == conditionType {
return false, nil
}
}
return true, nil
}
func FilterNodes(nodeList *v1.NodeList, fn func(node v1.Node) (test bool, err error)) {
var l []v1.Node
for _, node := range nodeList.Items {
test, err := fn(node)
if err != nil {
// FIXME error handling?
return
}
if test {
l = append(l, node)
}
}
nodeList.Items = l
}
func IsNodeConditionSetAsExpected(node *v1.Node, conditionType v1.NodeConditionType, wantTrue bool) (bool, error) {
return isNodeConditionSetAsExpected(node, conditionType, wantTrue, false)
}
// waitListSchedulableNodes is a wrapper around listing nodes supporting retries.
func (nodeAA *NodeAPIAdapter) waitListSchedulableNodes() (nodes *v1.NodeList, err error) {
if err = nodeAA.isClientDefined(); err != nil {
return nil, err
}
if wait.PollImmediate(Poll, SingleCallTimeout, func() (bool, error) {
c, err := nodeAA.getClient()
if err != nil {
// error logging TODO
return false, err
}
nodes, err = c.Nodes().List(v1.ListOptions{FieldSelector: "spec.unschedulable=false"})
if err != nil {
// error logging TODO
return false, err
}
return err == nil, nil
}) != nil {
// TODO logging
return nil, err
}
return nodes, err
}
func (nodeAA *NodeAPIAdapter) getClient() (*release_1_5.Clientset, error) {
// FIXME double check
if nodeAA.client == nil {
return nil, errors.New("Client cannot be null")
}
c := nodeAA.client.(*release_1_5.Clientset)
return c, nil
}
// TODO: remove slient bool ... but what is `wantTrue` defined as
func isNodeConditionSetAsExpected(node *v1.Node, conditionType v1.NodeConditionType, wantTrue, silent bool) (bool, error) {
if err := isNodeStatusDefined(node); err != nil {
return false, err
}
// Check the node readiness condition (logging all).
for _, cond := range node.Status.Conditions {
// Ensure that the condition type and the status matches as desired.
if cond.Type == conditionType {
if (cond.Status == v1.ConditionTrue) == wantTrue {
return true, nil
} else {
if !silent {
glog.V(4).Infof(
"Condition %s of node %s is %v instead of %t. Reason: %v, message: %v",
conditionType, node.Name, cond.Status == v1.ConditionTrue, wantTrue, cond.Reason, cond.Message)
}
return false, nil
}
}
}
if !silent {
glog.V(4).Infof("Couldn't find condition %v on node %v", conditionType, node.Name)
}
return false, nil
}
// Node is schedulable if:
// 1) doesn't have "unschedulable" field set
// 2) it's Ready condition is set to true
// 3) doesn't have NetworkUnavailable condition set to true
func isNodeSchedulable(node *v1.Node) (bool, error) {
nodeReady, err := IsNodeConditionSetAsExpected(node, v1.NodeReady, true)
if err != nil {
return false, err
}
networkUnval, err := IsNodeConditionUnset(node, v1.NodeNetworkUnavailable)
if err != nil {
return false, err
}
networkUnvalSilent, err := IsNodeConditionSetAsExpectedSilent(node, v1.NodeNetworkUnavailable, false)
if err != nil {
return false, err
}
networkReady := networkUnval || networkUnvalSilent
return !node.Spec.Unschedulable && nodeReady && networkReady, nil
}
func (nodeAA *NodeAPIAdapter) isNodeNameDefined() error {
if nodeAA.nodeName == "" {
return errors.New("nodeName must be defined in nodeAA struct")
}
return nil
}
func (nodeAA *NodeAPIAdapter) isClientDefined() error {
if nodeAA.client == nil {
return errors.New("client must be defined in the struct")
}
return nil
}
func isNodeStatusDefined(node *v1.Node) error {
if node == nil {
return errors.New("node cannot be nil")
}
// FIXME how do I test this?
/*
if node.Status == nil {
return errors.New("node.Status cannot be nil")
}*/
return nil
}
// Node is ready if:
// 1) it's Ready condition is set to true
// 2) doesn't have NetworkUnavailable condition set to true
func IsNodeOrMasterReady(node *v1.Node) (bool, error) {
nodeReady, err := IsNodeConditionSetAsExpected(node, v1.NodeReady, true)
if err != nil {
return false, err
}
networkUnval, err := IsNodeConditionUnset(node, v1.NodeNetworkUnavailable)
if err != nil {
return false, err
}
networkUnvalSilent, err := IsNodeConditionSetAsExpectedSilent(node, v1.NodeNetworkUnavailable, false)
if err != nil {
return false, err
}
networkReady := networkUnval || networkUnvalSilent
return nodeReady && networkReady, nil
}

View File

@ -0,0 +1,175 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package kops
import (
"testing"
//"time"
//"github.com/golang/glog"
//k8sapi "k8s.io/kubernetes/pkg/api"
//"k8s.io/kubernetes/pkg/client/unversioned/testclient"
//"k8s.io/kubernetes/pkg/client/unversioned/testclient/simple"
//"k8s.io/kubernetes/pkg/api/testapi"
)
func TestBuildNodeAPIAdapter(t *testing.T) {
}
func TestGetReadySchedulableNodes(t *testing.T) {
}
// TODO not working since they changed the darn api
/*
func TestWaitForNodeToBeReady(t *testing.T) {
conditions := make([]k8sapi.NodeCondition,1)
conditions[0] = k8sapi.NodeCondition{Type:"Ready",Status:"True"}
nodeAA := setupNodeAA(t,conditions)
test, err := nodeAA.WaitForNodeToBeReady()
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if test != true {
t.Fatalf("unexpected error WaitForNodeToBeReady Failed: %v", test)
}
}
func TestWaitForNodeToBeNotReady(t *testing.T) {
conditions := make([]k8sapi.NodeCondition,1)
conditions[0] = k8sapi.NodeCondition{Type:"Ready",Status:"False"}
nodeAA := setupNodeAA(t,conditions)
test, err := nodeAA.WaitForNodeToBeNotReady()
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if test != true {
t.Fatalf("unexpected error WaitForNodeToBeReady Failed: %v", test)
}
}
func TestIsNodeConditionUnset(t *testing.T) {
}
func setupNodeAA(t *testing.T, conditions []k8sapi.NodeCondition)(*NodeAPIAdapter) {
c := &simple.Client{
Request: simple.Request{
Method: "GET",
Path: testapi.Default.ResourcePath(getNodesResourceName(), "", "foo"),
},
Response: simple.Response{
StatusCode: 200,
Body: &k8sapi.Node{
ObjectMeta: k8sapi.ObjectMeta{Name: "node-foo"},
Spec: k8sapi.NodeSpec{ Unschedulable: false },
Status: k8sapi.NodeStatus{ Conditions: conditions},
},
},
}
c.Setup(t).Clientset.Nodes().Get("foo")
//c.Validate(t, response, err)
return &NodeAPIAdapter{
client: c.Clientset,
timeout: time.Duration(10)*time.Second,
nodeName: "foo",
}
}
*/
/*
func mockClient() *testclient.Fake {
return testclient.NewSimpleFake(dummyNode())
}
// Create a NodeAPIAdapter with K8s client based on the current kubectl config
func buildMockNodeAPIAdapter(nodeName string, t *testing.T) *NodeAPIAdapter {
s := simple.Client{}
c := s.Setup(t)
c.Nodes().Create(dummyNode())
node, err := c.Client.Nodes().Get("foo")
glog.V(4).Infof("node call %v, %v", node, err)
return &NodeAPIAdapter{
client: c.Client,
timeout: time.Duration(10)*time.Second,
nodeName: nodeName,
}
}
func dummyNode() *api.Node {
return &api.Node{
ObjectMeta: api.ObjectMeta{
Name: "foo",
},
Spec: api.NodeSpec{
Unschedulable: false,
},
}
}*/
func getNodesResourceName() string {
return "nodes"
}
/// Example mocking of api
/*
type secretsClient struct {
unversioned.Interface
}
// dummySecret generates a secret with one user inside the auth key
// foo:md5(bar)
func dummySecret() *api.Secret {
return &api.Secret{
ObjectMeta: api.ObjectMeta{
Namespace: api.NamespaceDefault,
Name: "demo-secret",
},
Data: map[string][]byte{"auth": []byte("foo:$apr1$OFG3Xybp$ckL0FHDAkoXYIlH9.cysT0")},
}
}
func mockClient() *testclient.Fake {
return testclient.NewSimpleFake(dummySecret())
}
func TestIngressWithoutAuth(t *testing.T) {
ing := buildIngress()
client := mockClient()
_, err := ParseAnnotations(client, ing, "")
if err == nil {
t.Error("Expected error with ingress without annotations")
}
if err == ErrMissingAuthType {
t.Errorf("Expected MissingAuthType error but returned %v", err)
}
}
*/

View File

@ -0,0 +1,176 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package kops
import (
"errors"
"time"
"fmt"
"github.com/golang/glog"
//client_simple "k8s.io/kops/pkg/client/simple"
//k8s_api "k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/api/v1"
)
const (
Node = "node"
Master = "master"
)
// A cluster to validate
type ValidationCluster struct {
MastersReady []*ValidationNode
MastersNotReady []*ValidationNode
// Leaving here if we are to determine which nodes are down
//MastersInstanceGroups []*api.InstanceGroup
MastersCount int
NodesReady []*ValidationNode
NodesNotReady []*ValidationNode
// Leaving here if we are to determine which nodes are down
//NodesInstanceGroups []*api.InstanceGroup
NodesCount int
}
// A K8s node to be validated
type ValidationNode struct {
Zone string
Role string
Hostname string
Status v1.ConditionStatus
}
// ValidateClusterWithIg validate a k8s clsuter with a provided instance group list
func ValidateClusterWithIg(clusterName string, instanceGroupList *InstanceGroupList) (*v1.NodeList, error) {
var instancegroups []*InstanceGroup
validationCluster := &ValidationCluster{}
for i := range instanceGroupList.Items {
ig := &instanceGroupList.Items[i]
instancegroups = append(instancegroups, ig)
if ig.Spec.Role == InstanceGroupRoleMaster {
// Leaving here if we are to determine which nodes are down
//validationCluster.mastersInstanceGroups = append(validationCluster.mastersInstanceGroups, ig)
validationCluster.MastersCount += *ig.Spec.MinSize
} else {
// Leaving here if we are to determine which nodes are down
//validationCluster.nodesInstanceGroups = append(validationCluster.nodesInstanceGroups, ig)
validationCluster.NodesCount += *ig.Spec.MinSize
}
}
nodeAA := &NodeAPIAdapter{}
timeout, err := time.ParseDuration("30s")
if err != nil {
return nil, fmt.Errorf("Cannot set timeout %q: %v", clusterName, err)
}
nodeAA.BuildNodeAPIAdapter(clusterName, timeout, "")
nodes, err := nodeAA.GetAllNodes()
if err != nil {
return nil, fmt.Errorf("Cannot get nodes for %q: %v", clusterName, err)
}
if len(instancegroups) == 0 {
return nodes, errors.New("No InstanceGroup objects found\n")
}
for _, node := range nodes.Items {
role := Node
if val, ok := node.ObjectMeta.Labels["kubernetes.io/role"]; ok {
role = val
}
n := &ValidationNode{
Zone: node.ObjectMeta.Labels["failure-domain.beta.kubernetes.io/zone"],
Hostname: node.ObjectMeta.Labels["kubernetes.io/hostname"],
Role: role,
Status: GetNodeConditionStatus(node.Status.Conditions),
}
ready, err := IsNodeOrMasterReady(&node)
if err != nil {
return nodes, fmt.Errorf("Cannot test if node is ready: %s", node.Name)
}
if n.Role == Master {
if ready {
validationCluster.MastersReady = append(validationCluster.MastersReady, n)
} else {
validationCluster.MastersNotReady = append(validationCluster.MastersNotReady, n)
}
} else if n.Role == Node {
if ready {
validationCluster.NodesReady = append(validationCluster.NodesReady, n)
} else {
validationCluster.NodesNotReady = append(validationCluster.NodesNotReady, n)
}
}
}
mastersReady := true
nodesReady := true
if len(validationCluster.MastersNotReady) != 0 || validationCluster.MastersCount !=
len(validationCluster.MastersReady) {
mastersReady = false
}
if len(validationCluster.NodesNotReady) != 0 || validationCluster.NodesCount !=
len(validationCluster.NodesReady) {
nodesReady = false
}
glog.Infof("validationCluster %+v", validationCluster)
if mastersReady && nodesReady {
return nodes, nil
} else {
return nodes, fmt.Errorf("You cluster is NOT ready %s", clusterName)
}
}
// ValidateCluster does what it is named, validate a K8s cluster
/*
func FullValidateCluster(clusterName string, clientset client_simple.Clientset) (*v1.NodeList, error) {
list, err := clientset.InstanceGroups(clusterName).List(k8s_api.ListOptions{})
if err != nil {
return nil, fmt.Errorf("Cannot get instnacegroups for %q: %v", clusterName, err)
}
return ValidateClusterWithIg(clusterName, list)
}*/
func GetNodeConditionStatus(nodeConditions []v1.NodeCondition) v1.ConditionStatus {
s := v1.ConditionUnknown
for _, element := range nodeConditions {
if element.Type == "Ready" {
s = element.Status
break
}
}
return s
}

View File

@ -17,13 +17,14 @@ limitations under the License.
package kutil
import (
"reflect"
"sort"
"testing"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/ec2"
"k8s.io/kops/cloudmock/aws/mockec2"
"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
"reflect"
"sort"
"testing"
)
func TestAddUntaggedRouteTables(t *testing.T) {

View File

@ -1,175 +0,0 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package kutil
import (
"github.com/golang/glog"
"k8s.io/kubernetes/pkg/api"
client "k8s.io/kubernetes/pkg/client/unversioned"
"k8s.io/kubernetes/pkg/fields"
"k8s.io/kubernetes/pkg/util/wait"
"time"
)
const (
// How often to Poll pods, nodes and claims.
Poll = 2 * time.Second
// How long to try single API calls (like 'get' or 'list'). Used to prevent
// transient failures from failing tests.
// TODO: client should not apply this timeout to Watch calls. Increased from 30s until that is fixed.
SingleCallTimeout = 5 * time.Minute
)
type NodeAPIAdapter struct {
client *client.Client
timeout time.Duration
}
// GetReadySchedulableNodesOrDie addresses the common use case of getting nodes you can do work on.
// 1) Needs to be schedulable.
// 2) Needs to be ready.
// If EITHER 1 or 2 is not true, most tests will want to ignore the node entirely.
func (nodeAA *NodeAPIAdapter) GetReadySchedulableNodes() (nodes *api.NodeList, err error) {
nodes, err = nodeAA.waitListSchedulableNodes()
if err != nil {
return nil, err
}
// previous tests may have cause failures of some nodes. Let's skip
// 'Not Ready' nodes, just in case (there is no need to fail the test).
FilterNodes(nodes, func(node api.Node) bool {
return isNodeSchedulable(&node), nil
})
return nodes, err
}
// WaitForNodeToBeReady returns whether node name is ready within timeout.
func (nodeAA *NodeAPIAdapter) WaitForNodeToBeReady(name string) bool {
return nodeAA.WaitForNodeToBe(name, api.NodeReady, true)
}
// WaitForNodeToBeNotReady returns whether node name is not ready (i.e. the
// readiness condition is anything but ready, e.g false or unknown) within
// timeout.
func (nodeAA *NodeAPIAdapter) WaitForNodeToBeNotReady(name string) bool {
return nodeAA.WaitForNodeToBe(name, api.NodeReady, false)
}
// WaitForNodeToBe returns whether node "name's" condition state matches wantTrue
// within timeout. If wantTrue is true, it will ensure the node condition status
// is ConditionTrue; if it's false, it ensures the node condition is in any state
// other than ConditionTrue (e.g. not true or unknown).
func (nodeAA *NodeAPIAdapter) WaitForNodeToBe(name string, conditionType api.NodeConditionType, wantTrue bool) bool {
glog.V(4).Infof("Waiting up to %v for node %s condition %s to be %t", nodeAA.timeout, name, conditionType, wantTrue)
for start := time.Now(); time.Since(start) < nodeAA.timeout; time.Sleep(Poll) {
node, err := nodeAA.client.Nodes().Get(name)
// FIXME this is not erroring on 500's for instance. We will keep looping
if err != nil {
glog.V(4).Infof("Couldn't get node %s", name)
continue
}
if IsNodeConditionSetAsExpected(node, conditionType, wantTrue) {
return true
}
}
glog.V(4).Infof("Node %s didn't reach desired %s condition status (%t) within %v", name, conditionType, wantTrue, nodeAA.timeout)
return false
}
func IsNodeConditionSetAsExpectedSilent(node *api.Node, conditionType api.NodeConditionType, wantTrue bool) bool {
return isNodeConditionSetAsExpected(node, conditionType, wantTrue, true)
}
func IsNodeConditionUnset(node *api.Node, conditionType api.NodeConditionType) bool {
for _, cond := range node.Status.Conditions {
if cond.Type == conditionType {
return false
}
}
return true
}
func FilterNodes(nodeList *api.NodeList, fn func(node api.Node) bool) {
var l []api.Node
for _, node := range nodeList.Items {
if fn(node) {
l = append(l, node)
}
}
nodeList.Items = l
}
func IsNodeConditionSetAsExpected(node *api.Node, conditionType api.NodeConditionType, wantTrue bool) bool {
return isNodeConditionSetAsExpected(node, conditionType, wantTrue, false)
}
// waitListSchedulableNodes is a wrapper around listing nodes supporting retries.
func (nodeAA *NodeAPIAdapter) waitListSchedulableNodes() (nodes *api.NodeList, err error) {
if wait.PollImmediate(Poll, SingleCallTimeout, func() (bool, error) {
nodes, err = nodeAA.client.Nodes().List(api.ListOptions{FieldSelector: fields.Set{
"spec.unschedulable": "false",
}.AsSelector()})
if err != nil {
return nil, err
}
return nodes, nil
}) != nil {
return nil, err
}
return nodes, nil
}
func isNodeConditionSetAsExpected(node *api.Node, conditionType api.NodeConditionType, wantTrue, silent bool) bool {
// Check the node readiness condition (logging all).
for _, cond := range node.Status.Conditions {
// Ensure that the condition type and the status matches as desired.
if cond.Type == conditionType {
if (cond.Status == api.ConditionTrue) == wantTrue {
return true
} else {
if !silent {
glog.V(4).Infof(
"Condition %s of node %s is %v instead of %t. Reason: %v, message: %v",
conditionType, node.Name, cond.Status == api.ConditionTrue, wantTrue, cond.Reason, cond.Message)
}
return false
}
}
}
if !silent {
glog.V(4).Infof("Couldn't find condition %v on node %v", conditionType, node.Name)
}
return false
}
// Node is schedulable if:
// 1) doesn't have "unschedulable" field set
// 2) it's Ready condition is set to true
// 3) doesn't have NetworkUnavailable condition set to true
func isNodeSchedulable(node *api.Node) bool {
nodeReady := IsNodeConditionSetAsExpected(node, api.NodeReady, true)
networkReady := IsNodeConditionUnset(node, api.NodeNetworkUnavailable) ||
IsNodeConditionSetAsExpectedSilent(node, api.NodeNetworkUnavailable, false)
return !node.Spec.Unschedulable && nodeReady && networkReady
}