kops/tests/e2e/pkg/tester/tester.go

/*
Copyright 2021 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package tester

import (
	"bytes"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/octago/sflags/gen/gpflag"
	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/klog/v2"
	"sigs.k8s.io/kubetest2/pkg/testers/ginkgo"

	api "k8s.io/kops/pkg/apis/kops/v1alpha2"
	"k8s.io/kops/tests/e2e/pkg/kops"
)

// Tester wraps kubetest2's ginkgo tester with additional functionality
type Tester struct {
	*ginkgo.Tester

	kopsCluster        *api.Cluster
	kopsInstanceGroups []*api.InstanceGroup
}

func (t *Tester) pretestSetup() error {
	kubectlPath, err := t.AcquireKubectl()
	if err != nil {
		return fmt.Errorf("failed to get kubectl package from published releases: %s", err)
	}

	existingPath := os.Getenv("PATH")
	newPath := fmt.Sprintf("%v:%v", filepath.Dir(kubectlPath), existingPath)
	klog.Info("Setting PATH=", newPath)
	return os.Setenv("PATH", newPath)
}

// parseKubeconfig will get the current kubeconfig, and extract the specified field by jsonpath.
func parseKubeconfig(jsonPath string) (string, error) {
	args := []string{
		"kubectl", "config", "view", "--minify", "-o", "jsonpath={" + jsonPath + "}",
	}
	c := exec.Command(args[0], args[1:]...)
	var stdout bytes.Buffer
	c.Stdout = &stdout
	var stderr bytes.Buffer
	c.Stderr = &stderr
	if err := c.Run(); err != nil {
		klog.Warningf("failed to run %s; stderr=%s", strings.Join(args, " "), stderr.String())
		return "", fmt.Errorf("error querying current config from kubectl: %w", err)
	}

	s := strings.TrimSpace(stdout.String())
	if s == "" {
		return "", fmt.Errorf("kubeconfig did not contain " + jsonPath)
	}
	return s, nil
}

// The --host flag was required in the kubernetes e2e tests, until https://github.com/kubernetes/kubernetes/pull/87030
// We can likely drop this when we drop support / testing for k8s 1.17
func (t *Tester) addHostFlag() error {
	server, err := parseKubeconfig(".clusters[0].cluster.server")
	if err != nil {
		return err
	}
	klog.Infof("Adding --host=%s", server)
	t.TestArgs += " --host=" + server
	return nil
}

// hasFlag detects if the specified flag has been passed in the args
func hasFlag(args string, flag string) bool {
	for _, arg := range strings.Split(args, " ") {
		if !strings.HasPrefix(arg, "-") {
			continue
		}

		arg = strings.TrimLeft(arg, "-")
		if arg == flag || strings.HasPrefix(arg, flag+"=") {
			return true
		}
	}
	return false
}

func (t *Tester) getKopsCluster() (*api.Cluster, error) {
	if t.kopsCluster != nil {
		return t.kopsCluster, nil
	}

	currentContext, err := parseKubeconfig(".current-context")
	if err != nil {
		return nil, err
	}

	kopsClusterName := currentContext

	cluster, err := kops.GetCluster(kopsClusterName)
	if err != nil {
		return nil, err
	}
	t.kopsCluster = cluster

	return cluster, nil

}

func (t *Tester) getKopsInstanceGroups() ([]*api.InstanceGroup, error) {
	if t.kopsInstanceGroups != nil {
		return t.kopsInstanceGroups, nil
	}

	cluster, err := t.getKopsCluster()
	if err != nil {
		return nil, err
	}

	igs, err := kops.GetInstanceGroups(cluster.Name)
	if err != nil {
		return nil, err
	}
	t.kopsInstanceGroups = igs

	return igs, nil

}
func (t *Tester) addProviderFlag() error {
	if hasFlag(t.TestArgs, "provider") {
		return nil
	}

	cluster, err := t.getKopsCluster()
	if err != nil {
		return err
	}

	provider := ""
	switch cluster.Spec.CloudProvider {
	case "aws", "gce":
		provider = cluster.Spec.CloudProvider
	default:
		return fmt.Errorf("unhandled cluster.spec.cloudProvider %q for determining ginkgo Provider", cluster.Spec.CloudProvider)
	}

	klog.Infof("Setting --provider=%s", provider)
	t.TestArgs += " --provider=" + provider
	return nil
}

func (t *Tester) addZoneFlag() error {
	// gce-zone is indeed used for AWS as well!
	if hasFlag(t.TestArgs, "gce-zone") {
		return nil
	}

	zoneNames, err := t.getZones()
	if err != nil {
		return err
	}

	// gce-zone only expects one zone, we just pass the first one
	zone := zoneNames[0]
	klog.Infof("Setting --gce-zone=%s", zone)
	t.TestArgs += " --gce-zone=" + zone

	// TODO: Pass the new gce-zones flag for 1.21 with all zones?

	return nil
}

func (t *Tester) addMultiZoneFlag() error {
	if hasFlag(t.TestArgs, "gce-multizone") {
		return nil
	}

	zoneNames, err := t.getZones()
	if err != nil {
		return err
	}

	klog.Infof("Setting --gce-multizone=%t", len(zoneNames) > 1)
	t.TestArgs += fmt.Sprintf(" --gce-multizone=%t", len(zoneNames) > 1)

	return nil
}

func (t *Tester) addRegionFlag() error {
	// gce-zone is used for other cloud providers as well
	if hasFlag(t.TestArgs, "gce-region") {
		return nil
	}

	cluster, err := t.getKopsCluster()
	if err != nil {
		return err
	}

	// We don't explicitly set the provider's region in the spec so we need to extract it from vairous fields
	var region string
	switch cluster.Spec.CloudProvider {
	case "aws":
		zone := cluster.Spec.Subnets[0].Zone
		region = zone[:len(zone)-1]
	case "gce":
		region = cluster.Spec.Subnets[0].Region
	default:
		return fmt.Errorf("unhandled region detection for cloud provider: %v", cluster.Spec.CloudProvider)
	}

	klog.Infof("Setting --gce-region=%s", region)
	t.TestArgs += " --gce-region=" + region
	return nil
}

func (t *Tester) addClusterTagFlag() error {
	if hasFlag(t.TestArgs, "cluster-tag") {
		return nil
	}

	cluster, err := t.getKopsCluster()
	if err != nil {
		return err
	}

	clusterName := cluster.ObjectMeta.Name
	klog.Infof("Setting --cluster-tag=%s", clusterName)
	t.TestArgs += " --cluster-tag=" + clusterName

	return nil
}

func (t *Tester) addProjectFlag() error {
	if hasFlag(t.TestArgs, "gce-project") {
		return nil
	}

	cluster, err := t.getKopsCluster()
	if err != nil {
		return err
	}

	projectID := cluster.Spec.Project
	if projectID == "" {
		return nil
	}
	klog.Infof("Setting --gce-project=%s", projectID)
	t.TestArgs += " --gce-project=" + projectID

	return nil
}

func (t *Tester) getZones() ([]string, error) {
	cluster, err := t.getKopsCluster()
	if err != nil {
		return nil, err
	}

	igs, err := t.getKopsInstanceGroups()
	if err != nil {
		return nil, err
	}

	zones := sets.NewString()
	// Gather zones on AWS
	for _, subnet := range cluster.Spec.Subnets {
		if subnet.Zone != "" {
			zones.Insert(subnet.Zone)
		}
	}
	// Gather zones on GCE
	for _, ig := range igs {
		for _, zone := range ig.Spec.Zones {
			zones.Insert(zone)
		}
	}
	zoneNames := zones.List()

	if len(zoneNames) == 0 {
		return nil, fmt.Errorf("no zones found in instance groups")
	}
	return zoneNames, nil
}

func (t *Tester) execute() error {
	fs, err := gpflag.Parse(t)
	if err != nil {
		return fmt.Errorf("failed to initialize tester: %v", err)
	}

	help := fs.BoolP("help", "h", false, "")
	if err := fs.Parse(os.Args); err != nil {
		return fmt.Errorf("failed to parse flags: %v", err)
	}

	if *help {
		fs.SetOutput(os.Stdout)
		fs.PrintDefaults()
		return nil
	}

	if err := t.pretestSetup(); err != nil {
		return err
	}

	if err := t.addHostFlag(); err != nil {
		return err
	}

	if err := t.addProviderFlag(); err != nil {
		return err
	}

	if err := t.addZoneFlag(); err != nil {
		return err
	}

	if err := t.addClusterTagFlag(); err != nil {
		return err
	}

	if err := t.addRegionFlag(); err != nil {
		return err
	}

	if err := t.addMultiZoneFlag(); err != nil {
		return err
	}

	if err := t.addProjectFlag(); err != nil {
		return err
	}

	return t.Test()
}

func NewDefaultTester() *Tester {
	t := &Tester{}
	t.Tester = ginkgo.NewDefaultTester()
	return t
}

func Main() {
	t := NewDefaultTester()
	if err := t.execute(); err != nil {
		klog.Fatalf("failed to run ginkgo tester: %v", err)
	}
}