kubernetes
/
kops
mirror of https://github.com/kubernetes/kops.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #10847 from justinsb/pass_provider_with_kubetest2 

kubetest2: Infer the provider and zones from the kops cluster
This commit is contained in:
Kubernetes Prow Robot 2021-04-27 08:57:38 -07:00 committed by GitHub
parent 6ed023ed30 038baef420
commit 3430c52fcc
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 1159 additions and 125 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

54
tests/e2e/go.mod
View File

@ -8,7 +8,61 @@ require (
github.com/octago/sflags v0.2.0
github.com/spf13/pflag v1.0.5
gopkg.in/yaml.v2 v2.4.0
k8s.io/apimachinery v0.21.0
k8s.io/klog/v2 v2.8.0
k8s.io/kops v0.0.0-00010101000000-000000000000
sigs.k8s.io/boskos v0.0.0-20200710214748-f5935686c7fc
sigs.k8s.io/kubetest2 v0.0.0-20210309183806-9230b4e73d8d
)
replace k8s.io/kops => ../../.
// These should match the go.mod from k8s.io/kops
replace k8s.io/api => k8s.io/api v0.21.0
replace k8s.io/apimachinery => k8s.io/apimachinery v0.21.0
replace k8s.io/client-go => k8s.io/client-go v0.21.0
replace k8s.io/cloud-provider => k8s.io/cloud-provider v0.21.0
replace k8s.io/controller-manager => k8s.io/controller-manager v0.21.0
replace k8s.io/kubectl => k8s.io/kubectl v0.21.0
replace k8s.io/apiserver => k8s.io/apiserver v0.21.0
replace k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.21.0
replace k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.21.0
replace k8s.io/kube-proxy => k8s.io/kube-proxy v0.21.0
replace k8s.io/cri-api => k8s.io/cri-api v0.21.0
replace k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.21.0
replace k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.21.0
replace k8s.io/component-base => k8s.io/component-base v0.21.0
replace k8s.io/component-helpers => k8s.io/component-helpers v0.21.0
replace k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.21.0
replace k8s.io/metrics => k8s.io/metrics v0.21.0
replace k8s.io/mount-utils => k8s.io/mount-utils v0.21.0
replace k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.21.0
replace k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.21.0
replace k8s.io/kubelet => k8s.io/kubelet v0.21.0
replace k8s.io/cli-runtime => k8s.io/cli-runtime v0.21.0
replace k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.21.0
replace k8s.io/code-generator => k8s.io/code-generator v0.21.0

693
tests/e2e/go.sum
View File

File diff suppressed because it is too large Load Diff

4
tests/e2e/kubetest2-kops/deployer/build.go
View File

@ -23,6 +23,7 @@ import (
"path"
"strings"
"k8s.io/kops/tests/e2e/pkg/util"
"sigs.k8s.io/kubetest2/pkg/exec"
)
@ -38,7 +39,8 @@ func (d *deployer) Build() error {
if err := d.BuildOptions.Build(); err != nil {
return err
}
return nil
// Copy the kops binary into the test's RunDir to be included in the tester's PATH
return util.Copy(d.KopsBinaryPath, path.Join(d.commonOptions.RunDir(), "kops"))
}
func (d *deployer) verifyBuildFlags() error {

21
tests/e2e/kubetest2-kops/deployer/common.go
View File

@ -57,13 +57,14 @@ func (d *deployer) initialize() error {
}
}
if d.KopsVersionMarker != "" {
binaryPath, baseURL, err := kops.DownloadKops(d.KopsVersionMarker, d.KopsBinaryPath)
d.KopsBinaryPath = path.Join(d.commonOptions.RunDir(), "kops")
baseURL, err := kops.DownloadKops(d.KopsVersionMarker, d.KopsBinaryPath)
if err != nil {
return fmt.Errorf("init failed to download kops from url: %v", err)
}
d.KopsBinaryPath = binaryPath
d.KopsBaseURL = baseURL
}
switch d.CloudProvider {
case "aws":
// These environment variables are defined by the "preset-aws-ssh" prow preset
@ -105,7 +106,7 @@ func (d *deployer) initialize() error {
d.SSHPrivateKeyPath = privateKey
d.SSHPublicKeyPath = publicKey
}
d.createBucket = os.Getenv("KOPS_STATE_STORE") == ""
d.createBucket = true
}
}
if d.SSHUser == "" {
@ -118,6 +119,14 @@ func (d *deployer) initialize() error {
}
d.terraform = t
}
if d.commonOptions.ShouldTest() {
for _, envvar := range d.env() {
// Set all of the env vars we use for kops in the current process
// so that the tester inherits them when shelling out to kops
i := strings.Index(envvar, "=")
os.Setenv(envvar[0:i], envvar[i+1:])
}
}
return nil
}
@ -133,11 +142,7 @@ func (d *deployer) verifyKopsFlags() error {
}
if d.KopsBinaryPath == "" && d.KopsVersionMarker == "" {
if ws := os.Getenv("WORKSPACE"); ws != "" {
d.KopsBinaryPath = path.Join(ws, "kops")
} else {
return errors.New("missing required --kops-binary-path when --kops-version-marker is not used")
}
return errors.New("missing required --kops-binary-path when --kops-version-marker is not used")
}
switch d.CloudProvider {

30
tests/e2e/pkg/kops/download.go
View File

@ -27,39 +27,29 @@ import (
)
// DownloadKops will download the kops binary from the version marker URL
// Returning the path to the local kops binary and the URL to use for KOPS_BASE_URL
// Returning the URL to use for KOPS_BASE_URL
// Example markerURL: https://storage.googleapis.com/kops-ci/bin/latest-ci-updown-green.txt
func DownloadKops(markerURL, downloadPath string) (string, string, error) {
func DownloadKops(markerURL, downloadPath string) (string, error) {
var b bytes.Buffer
if err := util.HTTPGETWithHeaders(markerURL, nil, &b); err != nil {
return "", "", err
return "", err
}
kopsBaseURL := strings.TrimSpace(b.String())
var kopsFile *os.File
if downloadPath == "" {
tmp, err := os.CreateTemp("", "kops")
if err != nil {
return "", "", err
}
kopsFile = tmp
} else {
tmp, err := os.Create(downloadPath)
if err != nil {
return "", "", err
}
kopsFile = tmp
kopsFile, err := os.Create(downloadPath)
if err != nil {
return "", err
}
kopsURL := fmt.Sprintf("%v/%v/%v/kops", kopsBaseURL, runtime.GOOS, runtime.GOARCH)
if err := util.HTTPGETWithHeaders(kopsURL, nil, kopsFile); err != nil {
return "", "", err
return "", err
}
if err := kopsFile.Close(); err != nil {
return "", "", err
return "", err
}
if err := os.Chmod(kopsFile.Name(), 0755); err != nil {
return "", "", err
return "", err
}
return kopsFile.Name(), kopsBaseURL, nil
return kopsBaseURL, nil
}

72
tests/e2e/pkg/kops/state.go Normal file
View File

@ -0,0 +1,72 @@
/*
Copyright 2021 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package kops
import (
"bytes"
"encoding/json"
"fmt"
"os/exec"
"strings"
"k8s.io/klog/v2"
api "k8s.io/kops/pkg/apis/kops/v1alpha2"
)
// GetCluster will retrieve the specified Cluster from the state store.
func GetCluster(clusterName string) (*api.Cluster, error) {
args := []string{
"kops", "get", "cluster", clusterName, "-ojson",
}
c := exec.Command(args[0], args[1:]...)
var stdout bytes.Buffer
c.Stdout = &stdout
var stderr bytes.Buffer
c.Stderr = &stderr
if err := c.Run(); err != nil {
klog.Warningf("failed to run %s; stderr=%s", strings.Join(args, " "), stderr.String())
return nil, fmt.Errorf("error querying cluster from %s: %w", strings.Join(args, " "), err)
}
cluster := &api.Cluster{}
if err := json.Unmarshal(stdout.Bytes(), cluster); err != nil {
return nil, fmt.Errorf("error parsing cluster json: %w", err)
}
return cluster, nil
}
// GetInstanceGroups will retrieve the instance groups for the specified Cluster from the state store.
func GetInstanceGroups(clusterName string) ([]*api.InstanceGroup, error) {
args := []string{
"kops", "get", "instancegroups", "--name", clusterName, "-ojson",
}
c := exec.Command(args[0], args[1:]...)
var stdout bytes.Buffer
c.Stdout = &stdout
var stderr bytes.Buffer
c.Stderr = &stderr
if err := c.Run(); err != nil {
klog.Warningf("failed to run %s; stderr=%s", strings.Join(args, " "), stderr.String())
return nil, fmt.Errorf("error querying instance groups from %s: %w", strings.Join(args, " "), err)
}
var igs []*api.InstanceGroup
if err := json.Unmarshal(stdout.Bytes(), &igs); err != nil {
return nil, fmt.Errorf("error parsing instance groups json: %w", err)
}
return igs, nil
}

277
tests/e2e/pkg/tester/tester.go
View File

@ -25,14 +25,20 @@ import (
"strings"
"github.com/octago/sflags/gen/gpflag"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/klog/v2"
"sigs.k8s.io/kubetest2/pkg/testers/ginkgo"
api "k8s.io/kops/pkg/apis/kops/v1alpha2"
"k8s.io/kops/tests/e2e/pkg/kops"
)
// Tester wraps kubetest2's ginkgo tester with additional functionality
type Tester struct {
*ginkgo.Tester
kopsCluster *api.Cluster
kopsInstanceGroups []*api.InstanceGroup
}
func (t *Tester) pretestSetup() error {
@ -47,11 +53,10 @@ func (t *Tester) pretestSetup() error {
return os.Setenv("PATH", newPath)
}
// The --host argument was required in the kubernetes e2e tests, until https://github.com/kubernetes/kubernetes/pull/87030
// We can likely drop this when we drop support / testing for k8s 1.17
func (t *Tester) addHostArgument() error {
// parseKubeconfig will get the current kubeconfig, and extract the specified field by jsonpath.
func parseKubeconfig(jsonPath string) (string, error) {
args := []string{
"kubectl", "config", "view", "--minify", "-o", "jsonpath='{.clusters[0].cluster.server}'",
"kubectl", "config", "view", "--minify", "-o", "jsonpath={" + jsonPath + "}",
}
c := exec.Command(args[0], args[1:]...)
var stdout bytes.Buffer
@ -60,19 +65,241 @@ func (t *Tester) addHostArgument() error {
c.Stderr = &stderr
if err := c.Run(); err != nil {
klog.Warningf("failed to run %s; stderr=%s", strings.Join(args, " "), stderr.String())
return fmt.Errorf("error querying current config from kubectl: %w", err)
return "", fmt.Errorf("error querying current config from kubectl: %w", err)
}
server := strings.TrimSpace(stdout.String())
if server == "" {
return fmt.Errorf("kubeconfig did not contain server")
s := strings.TrimSpace(stdout.String())
if s == "" {
return "", fmt.Errorf("kubeconfig did not contain " + jsonPath)
}
return s, nil
}
// The --host flag was required in the kubernetes e2e tests, until https://github.com/kubernetes/kubernetes/pull/87030
// We can likely drop this when we drop support / testing for k8s 1.17
func (t *Tester) addHostFlag() error {
server, err := parseKubeconfig(".clusters[0].cluster.server")
if err != nil {
return err
}
klog.Infof("Adding --host=%s", server)
t.TestArgs += " --host=" + server
return nil
}
// hasFlag detects if the specified flag has been passed in the args
func hasFlag(args string, flag string) bool {
for _, arg := range strings.Split(args, " ") {
if !strings.HasPrefix(arg, "-") {
continue
}
arg = strings.TrimLeft(arg, "-")
if arg == flag || strings.HasPrefix(arg, flag+"=") {
return true
}
}
return false
}
func (t *Tester) getKopsCluster() (*api.Cluster, error) {
if t.kopsCluster != nil {
return t.kopsCluster, nil
}
currentContext, err := parseKubeconfig(".current-context")
if err != nil {
return nil, err
}
kopsClusterName := currentContext
cluster, err := kops.GetCluster(kopsClusterName)
if err != nil {
return nil, err
}
t.kopsCluster = cluster
return cluster, nil
}
func (t *Tester) getKopsInstanceGroups() ([]*api.InstanceGroup, error) {
if t.kopsInstanceGroups != nil {
return t.kopsInstanceGroups, nil
}
cluster, err := t.getKopsCluster()
if err != nil {
return nil, err
}
igs, err := kops.GetInstanceGroups(cluster.Name)
if err != nil {
return nil, err
}
t.kopsInstanceGroups = igs
return igs, nil
}
func (t *Tester) addProviderFlag() error {
if hasFlag(t.TestArgs, "provider") {
return nil
}
cluster, err := t.getKopsCluster()
if err != nil {
return err
}
provider := ""
switch cluster.Spec.CloudProvider {
case "aws", "gce":
provider = cluster.Spec.CloudProvider
default:
return fmt.Errorf("unhandled cluster.spec.cloudProvider %q for determining ginkgo Provider", cluster.Spec.CloudProvider)
}
klog.Infof("Setting --provider=%s", provider)
t.TestArgs += " --provider=" + provider
return nil
}
func (t *Tester) addZoneFlag() error {
// gce-zone is indeed used for AWS as well!
if hasFlag(t.TestArgs, "gce-zone") {
return nil
}
zoneNames, err := t.getZones()
if err != nil {
return err
}
// gce-zone only expects one zone, we just pass the first one
zone := zoneNames[0]
klog.Infof("Setting --gce-zone=%s", zone)
t.TestArgs += " --gce-zone=" + zone
// TODO: Pass the new gce-zones flag for 1.21 with all zones?
return nil
}
func (t *Tester) addMultiZoneFlag() error {
if hasFlag(t.TestArgs, "gce-multizone") {
return nil
}
zoneNames, err := t.getZones()
if err != nil {
return err
}
klog.Infof("Setting --gce-multizone=%t", len(zoneNames) > 1)
t.TestArgs += fmt.Sprintf(" --gce-multizone=%t", len(zoneNames) > 1)
return nil
}
func (t *Tester) addRegionFlag() error {
// gce-zone is used for other cloud providers as well
if hasFlag(t.TestArgs, "gce-region") {
return nil
}
cluster, err := t.getKopsCluster()
if err != nil {
return err
}
// We don't explicitly set the provider's region in the spec so we need to extract it from vairous fields
var region string
switch cluster.Spec.CloudProvider {
case "aws":
zone := cluster.Spec.Subnets[0].Zone
region = zone[:len(zone)-1]
case "gce":
region = cluster.Spec.Subnets[0].Region
default:
return fmt.Errorf("unhandled region detection for cloud provider: %v", cluster.Spec.CloudProvider)
}
klog.Infof("Setting --gce-region=%s", region)
t.TestArgs += " --gce-region=" + region
return nil
}
func (t *Tester) addClusterTagFlag() error {
if hasFlag(t.TestArgs, "cluster-tag") {
return nil
}
cluster, err := t.getKopsCluster()
if err != nil {
return err
}
clusterName := cluster.ObjectMeta.Name
klog.Infof("Setting --cluster-tag=%s", clusterName)
t.TestArgs += " --cluster-tag=" + clusterName
return nil
}
func (t *Tester) addProjectFlag() error {
if hasFlag(t.TestArgs, "gce-project") {
return nil
}
cluster, err := t.getKopsCluster()
if err != nil {
return err
}
projectID := cluster.Spec.Project
if projectID == "" {
return nil
}
klog.Infof("Setting --gce-project=%s", projectID)
t.TestArgs += " --gce-project=" + projectID
return nil
}
func (t *Tester) getZones() ([]string, error) {
cluster, err := t.getKopsCluster()
if err != nil {
return nil, err
}
igs, err := t.getKopsInstanceGroups()
if err != nil {
return nil, err
}
zones := sets.NewString()
// Gather zones on AWS
for _, subnet := range cluster.Spec.Subnets {
if subnet.Zone != "" {
zones.Insert(subnet.Zone)
}
}
// Gather zones on GCE
for _, ig := range igs {
for _, zone := range ig.Spec.Zones {
zones.Insert(zone)
}
}
zoneNames := zones.List()
if len(zoneNames) == 0 {
return nil, fmt.Errorf("no zones found in instance groups")
}
return zoneNames, nil
}
func (t *Tester) execute() error {
fs, err := gpflag.Parse(t)
if err != nil {
@ -94,7 +321,31 @@ func (t *Tester) execute() error {
return err
}
if err := t.addHostArgument(); err != nil {
if err := t.addHostFlag(); err != nil {
return err
}
if err := t.addProviderFlag(); err != nil {
return err
}
if err := t.addZoneFlag(); err != nil {
return err
}
if err := t.addClusterTagFlag(); err != nil {
return err
}
if err := t.addRegionFlag(); err != nil {
return err
}
if err := t.addMultiZoneFlag(); err != nil {
return err
}
if err := t.addProjectFlag(); err != nil {
return err
}
@ -102,9 +353,9 @@ func (t *Tester) execute() error {
}
func NewDefaultTester() *Tester {
return &Tester{
ginkgo.NewDefaultTester(),
}
t := &Tester{}
t.Tester = ginkgo.NewDefaultTester()
return t
}
func Main() {

95
tests/e2e/pkg/tester/tester_test.go Normal file
View File

@ -0,0 +1,95 @@
/*
Copyright 2021 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package tester
import (
"testing"
"github.com/octago/sflags/gen/gpflag"
)
func TestFlagParsing(t *testing.T) {
tester := &Tester{}
fs, err := gpflag.Parse(tester)
if err != nil {
t.Fatalf("gpflag.Parse(tester) failed: %v", err)
}
args := []string{"--parallel", "25"}
if err := fs.Parse(args); err != nil {
t.Fatalf("fs.Parse(args) failed: %v", err)
}
if tester.Parallel != 25 {
t.Errorf("unexpected value for Parallel; got %d, want %d", tester.Parallel, 25)
}
}
func TestHasFlag(t *testing.T) {
grid := []struct {
Args string
Flag string
Expected bool
}{
{
Args: "--provider aws",
Flag: "provider",
Expected: true,
},
{
Args: "-provider aws",
Flag: "provider",
Expected: true,
},
{
Args: "provider aws",
Flag: "provider",
Expected: false,
},
{
Args: "-provider=aws",
Flag: "provider",
Expected: true,
},
{
Args: "--provider=aws",
Flag: "provider",
Expected: true,
},
{
Args: "--foo=bar --provider aws",
Flag: "provider",
Expected: true,
},
{
Args: "--foo=bar",
Flag: "provider",
Expected: false,
},
}
for _, g := range grid {
t.Run(g.Args, func(t *testing.T) {
got := hasFlag(g.Args, g.Flag)
if got != g.Expected {
t.Errorf("hasFlags(%q, %q) got %v, want %v", g.Args, g.Flag, got, g.Expected)
}
})
}
}

38
tests/e2e/pkg/util/copy.go Normal file
View File

@ -0,0 +1,38 @@
/*
Copyright 2021 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package util
import (
"strings"
"k8s.io/klog/v2"
"sigs.k8s.io/kubetest2/pkg/exec"
)
func Copy(src, dest string) error {
cpArgs := []string{
"cp", src, dest,
}
klog.Info(strings.Join(cpArgs, " "))
cmd := exec.Command(cpArgs[0], cpArgs[1:]...)
exec.InheritOutput(cmd)
err := cmd.Run()
if err != nil {
return err
}
return nil
}