kubernetes
/
kops
mirror of https://github.com/kubernetes/kops.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

kubetest2: Infer the provider and zones from the kops cluster 

This means we don't need to pass these flags explicitly.
This commit is contained in:
Justin SB 2021-02-16 08:00:07 -05:00 committed by Peter Rifel
parent 6ed023ed30
commit b21df766fc
No known key found for this signature in database
GPG Key ID: BC6469E5B16DB2B6
5 changed files with 986 additions and 96 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

54
tests/e2e/go.mod
View File

@ -8,7 +8,61 @@ require (
github.com/octago/sflags v0.2.0
github.com/spf13/pflag v1.0.5
gopkg.in/yaml.v2 v2.4.0
k8s.io/apimachinery v0.21.0
k8s.io/klog/v2 v2.8.0
k8s.io/kops v0.0.0-00010101000000-000000000000
sigs.k8s.io/boskos v0.0.0-20200710214748-f5935686c7fc
sigs.k8s.io/kubetest2 v0.0.0-20210309183806-9230b4e73d8d
)
replace k8s.io/kops => ../../.
// These should match the go.mod from k8s.io/kops
replace k8s.io/api => k8s.io/api v0.21.0
replace k8s.io/apimachinery => k8s.io/apimachinery v0.21.0
replace k8s.io/client-go => k8s.io/client-go v0.21.0
replace k8s.io/cloud-provider => k8s.io/cloud-provider v0.21.0
replace k8s.io/controller-manager => k8s.io/controller-manager v0.21.0
replace k8s.io/kubectl => k8s.io/kubectl v0.21.0
replace k8s.io/apiserver => k8s.io/apiserver v0.21.0
replace k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.21.0
replace k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.21.0
replace k8s.io/kube-proxy => k8s.io/kube-proxy v0.21.0
replace k8s.io/cri-api => k8s.io/cri-api v0.21.0
replace k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.21.0
replace k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.21.0
replace k8s.io/component-base => k8s.io/component-base v0.21.0
replace k8s.io/component-helpers => k8s.io/component-helpers v0.21.0
replace k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.21.0
replace k8s.io/metrics => k8s.io/metrics v0.21.0
replace k8s.io/mount-utils => k8s.io/mount-utils v0.21.0
replace k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.21.0
replace k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.21.0
replace k8s.io/kubelet => k8s.io/kubelet v0.21.0
replace k8s.io/cli-runtime => k8s.io/cli-runtime v0.21.0
replace k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.21.0
replace k8s.io/code-generator => k8s.io/code-generator v0.21.0

689
tests/e2e/go.sum
View File

File diff suppressed because it is too large Load Diff

72
tests/e2e/pkg/kops/state.go Normal file
View File

@ -0,0 +1,72 @@
/*
Copyright 2021 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package kops
import (
"bytes"
"encoding/json"
"fmt"
"os/exec"
"strings"
"k8s.io/klog/v2"
api "k8s.io/kops/pkg/apis/kops/v1alpha2"
)
// GetCluster will retrieve the specified Cluster from the state store.
func GetCluster(clusterName string) (*api.Cluster, error) {
args := []string{
"kops", "get", "cluster", clusterName, "-ojson",
}
c := exec.Command(args[0], args[1:]...)
var stdout bytes.Buffer
c.Stdout = &stdout
var stderr bytes.Buffer
c.Stderr = &stderr
if err := c.Run(); err != nil {
klog.Warningf("failed to run %s; stderr=%s", strings.Join(args, " "), stderr.String())
return nil, fmt.Errorf("error querying cluster from %s: %w", strings.Join(args, " "), err)
}
cluster := &api.Cluster{}
if err := json.Unmarshal(stdout.Bytes(), cluster); err != nil {
return nil, fmt.Errorf("error parsing cluster json: %w", err)
}
return cluster, nil
}
// GetInstanceGroups will retrieve the instance groups for the specified Cluster from the state store.
func GetInstanceGroups(clusterName string) ([]*api.InstanceGroup, error) {
args := []string{
"kops", "get", "instancegroups", "--name", clusterName, "-ojson",
}
c := exec.Command(args[0], args[1:]...)
var stdout bytes.Buffer
c.Stdout = &stdout
var stderr bytes.Buffer
c.Stderr = &stderr
if err := c.Run(); err != nil {
klog.Warningf("failed to run %s; stderr=%s", strings.Join(args, " "), stderr.String())
return nil, fmt.Errorf("error querying instance groups from %s: %w", strings.Join(args, " "), err)
}
var igs []*api.InstanceGroup
if err := json.Unmarshal(stdout.Bytes(), &igs); err != nil {
return nil, fmt.Errorf("error parsing instance groups json: %w", err)
}
return igs, nil
}

172
tests/e2e/pkg/tester/tester.go
View File

@ -25,14 +25,20 @@ import (
"strings"
"github.com/octago/sflags/gen/gpflag"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/klog/v2"
"sigs.k8s.io/kubetest2/pkg/testers/ginkgo"
api "k8s.io/kops/pkg/apis/kops/v1alpha2"
"k8s.io/kops/tests/e2e/pkg/kops"
)
// Tester wraps kubetest2's ginkgo tester with additional functionality
type Tester struct {
*ginkgo.Tester
kopsCluster *api.Cluster
kopsInstanceGroups []*api.InstanceGroup
}
func (t *Tester) pretestSetup() error {
@ -47,11 +53,10 @@ func (t *Tester) pretestSetup() error {
return os.Setenv("PATH", newPath)
}
// The --host argument was required in the kubernetes e2e tests, until https://github.com/kubernetes/kubernetes/pull/87030
// We can likely drop this when we drop support / testing for k8s 1.17
func (t *Tester) addHostArgument() error {
// parseKubeconfig will get the current kubeconfig, and extract the specified field by jsonpath.
func parseKubeconfig(jsonPath string) (string, error) {
args := []string{
"kubectl", "config", "view", "--minify", "-o", "jsonpath='{.clusters[0].cluster.server}'",
"kubectl", "config", "view", "--minify", "-o", "jsonpath={" + jsonPath + "}",
}
c := exec.Command(args[0], args[1:]...)
var stdout bytes.Buffer
@ -60,19 +65,152 @@ func (t *Tester) addHostArgument() error {
c.Stderr = &stderr
if err := c.Run(); err != nil {
klog.Warningf("failed to run %s; stderr=%s", strings.Join(args, " "), stderr.String())
return fmt.Errorf("error querying current config from kubectl: %w", err)
return "", fmt.Errorf("error querying current config from kubectl: %w", err)
}
server := strings.TrimSpace(stdout.String())
if server == "" {
return fmt.Errorf("kubeconfig did not contain server")
s := strings.TrimSpace(stdout.String())
if s == "" {
return "", fmt.Errorf("kubeconfig did not contain " + jsonPath)
}
return s, nil
}
// The --host flag was required in the kubernetes e2e tests, until https://github.com/kubernetes/kubernetes/pull/87030
// We can likely drop this when we drop support / testing for k8s 1.17
func (t *Tester) addHostFlag() error {
server, err := parseKubeconfig(".clusters[0].cluster.server")
if err != nil {
return err
}
klog.Infof("Adding --host=%s", server)
t.TestArgs += " --host=" + server
return nil
}
// hasFlag detects if the specified flag has been passed in the args
func hasFlag(args string, flag string) bool {
for _, arg := range strings.Split(args, " ") {
if !strings.HasPrefix(arg, "-") {
continue
}
arg = strings.TrimLeft(arg, "-")
if arg == flag || strings.HasPrefix(arg, flag+"=") {
return true
}
}
return false
}
func (t *Tester) getKopsCluster() (*api.Cluster, error) {
if t.kopsCluster != nil {
return t.kopsCluster, nil
}
currentContext, err := parseKubeconfig(".current-context")
if err != nil {
return nil, err
}
kopsClusterName := currentContext
cluster, err := kops.GetCluster(kopsClusterName)
if err != nil {
return nil, err
}
t.kopsCluster = cluster
return cluster, nil
}
func (t *Tester) getKopsInstanceGroups() ([]*api.InstanceGroup, error) {
if t.kopsInstanceGroups != nil {
return t.kopsInstanceGroups, nil
}
cluster, err := t.getKopsCluster()
if err != nil {
return nil, err
}
igs, err := kops.GetInstanceGroups(cluster.Name)
if err != nil {
return nil, err
}
t.kopsInstanceGroups = igs
return igs, nil
}
func (t *Tester) addProviderFlag() error {
if hasFlag(t.TestArgs, "provider") {
return nil
}
cluster, err := t.getKopsCluster()
if err != nil {
return err
}
provider := ""
switch cluster.Spec.CloudProvider {
case "aws", "gce":
provider = cluster.Spec.CloudProvider
default:
return fmt.Errorf("unhandled cluster.spec.cloudProvider %q for determining ginkgo Provider", cluster.Spec.CloudProvider)
}
klog.Infof("Setting --provider=%s", provider)
t.TestArgs += " --provider=" + provider
return nil
}
func (t *Tester) addZoneFlag() error {
// gce-zone is indeed used for AWS as well!
if hasFlag(t.TestArgs, "gce-zone") {
return nil
}
cluster, err := t.getKopsCluster()
if err != nil {
return err
}
igs, err := t.getKopsInstanceGroups()
if err != nil {
return err
}
zones := sets.NewString()
// Gather zones on AWS
for _, subnet := range cluster.Spec.Subnets {
if subnet.Zone != "" {
zones.Insert(subnet.Zone)
}
}
// Gather zones on GCE
for _, ig := range igs {
for _, zone := range ig.Spec.Zones {
zones.Insert(zone)
}
}
zoneNames := zones.List()
if len(zoneNames) == 0 {
return fmt.Errorf("no zones found in instance groups")
}
// gce-zone only expects one zone, we just pass the first one
zone := zoneNames[0]
klog.Infof("Setting --gce-zone=%s", zone)
t.TestArgs += " --gce-zone=" + zone
// TODO: Pass the new gce-zones flag for 1.21 with all zones?
return nil
}
func (t *Tester) execute() error {
fs, err := gpflag.Parse(t)
if err != nil {
@ -94,7 +232,15 @@ func (t *Tester) execute() error {
return err
}
if err := t.addHostArgument(); err != nil {
if err := t.addHostFlag(); err != nil {
return err
}
if err := t.addProviderFlag(); err != nil {
return err
}
if err := t.addZoneFlag(); err != nil {
return err
}
@ -102,9 +248,9 @@ func (t *Tester) execute() error {
}
func NewDefaultTester() *Tester {
return &Tester{
ginkgo.NewDefaultTester(),
}
t := &Tester{}
t.Tester = ginkgo.NewDefaultTester()
return t
}
func Main() {

95
tests/e2e/pkg/tester/tester_test.go Normal file
View File

@ -0,0 +1,95 @@
/*
Copyright 2021 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package tester
import (
"testing"
"github.com/octago/sflags/gen/gpflag"
)
func TestFlagParsing(t *testing.T) {
tester := &Tester{}
fs, err := gpflag.Parse(tester)
if err != nil {
t.Fatalf("gpflag.Parse(tester) failed: %v", err)
}
args := []string{"--parallel", "25"}
if err := fs.Parse(args); err != nil {
t.Fatalf("fs.Parse(args) failed: %v", err)
}
if tester.Parallel != 25 {
t.Errorf("unexpected value for Parallel; got %d, want %d", tester.Parallel, 25)
}
}
func TestHasFlag(t *testing.T) {
grid := []struct {
Args string
Flag string
Expected bool
}{
{
Args: "--provider aws",
Flag: "provider",
Expected: true,
},
{
Args: "-provider aws",
Flag: "provider",
Expected: true,
},
{
Args: "provider aws",
Flag: "provider",
Expected: false,
},
{
Args: "-provider=aws",
Flag: "provider",
Expected: true,
},
{
Args: "--provider=aws",
Flag: "provider",
Expected: true,
},
{
Args: "--foo=bar --provider aws",
Flag: "provider",
Expected: true,
},
{
Args: "--foo=bar",
Flag: "provider",
Expected: false,
},
}
for _, g := range grid {
t.Run(g.Args, func(t *testing.T) {
got := hasFlag(g.Args, g.Flag)
if got != g.Expected {
t.Errorf("hasFlags(%q, %q) got %v, want %v", g.Args, g.Flag, got, g.Expected)
}
})
}
}