kubernetes
/
kops
mirror of https://github.com/kubernetes/kops.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #10083 from rifelpet/kubetest2 

kubetest2 - Implement create/validate/delete cluster functionality
This commit is contained in:
Kubernetes Prow Robot 2020-10-24 01:11:00 -07:00 committed by GitHub
parent d7591f4b7f 676c6f5a7f
commit 51511ba59b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 386 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Makefile
View File

@ -384,6 +384,7 @@ gomod: gomod-prereqs
find vendor/ -name "BUILD" -delete
find vendor/ -name "BUILD.bazel" -delete
make gazelle
cd tests/e2e; GO111MODULE=on go mod tidy
.PHONY: gofmt

2
hack/verify-gomod.sh
View File

@ -24,7 +24,7 @@ cd "${KOPS_ROOT}"
make gomod
changes=$(git status --porcelain go.mod go.sum vendor/ || true)
changes=$(git status --porcelain go.mod go.sum vendor/ tests/e2e/go.mod tests/e2e/go.sum || true)
if [ -n "${changes}" ]; then
echo "ERROR: go modules are not up to date; please run: make gomod"
echo "changed files:"

112
tests/e2e/kubetest2-kops/deployer/common.go
View File

@ -16,7 +16,15 @@ limitations under the License.
package deployer
import "fmt"
import (
"crypto/md5"
"errors"
"fmt"
"os"
"path"
"k8s.io/klog/v2"
)
func (d *deployer) init() error {
var err error
@ -28,8 +36,108 @@ func (d *deployer) init() error {
func (d *deployer) initialize() error {
if d.commonOptions.ShouldBuild() {
if err := d.verifyBuildFlags(); err != nil {
return fmt.Errorf("init failed to check build flags: %s", err)
return fmt.Errorf("init failed to check build flags: %v", err)
}
}
if d.commonOptions.ShouldUp() || d.commonOptions.ShouldDown() {
if err := d.verifyKopsFlags(); err != nil {
return fmt.Errorf("init failed to check kops flags: %v", err)
}
}
if d.commonOptions.ShouldUp() {
if err := d.verifyUpFlags(); err != nil {
return fmt.Errorf("init failed to check up flags: %v", err)
}
}
return nil
}
// verifyKopsFlags ensures common fields are set for kops commands
func (d *deployer) verifyKopsFlags() error {
if d.ClusterName == "" {
name, err := defaultClusterName(d.CloudProvider)
if err != nil {
return err
}
klog.Info("Using cluster name ", d.ClusterName)
d.ClusterName = name
}
if d.KopsBinaryPath == "" {
if ws := os.Getenv("WORKSPACE"); ws != "" {
d.KopsBinaryPath = path.Join(ws, "kops")
} else {
return errors.New("missing required --kops-binary-path")
}
}
_, err := os.Stat(d.KopsBinaryPath)
if err != nil {
return err
}
switch d.CloudProvider {
case "aws":
default:
return errors.New("unsupported --cloud-provider value")
}
return nil
}
// env returns a list of environment variables passed to the kops binary
func (d *deployer) env() []string {
vars := d.Env
vars = append(vars, []string{
fmt.Sprintf("PATH=%v", os.Getenv("PATH")),
fmt.Sprintf("HOME=%v", os.Getenv("HOME")),
fmt.Sprintf("KOPS_STATE_STORE=%v", stateStore(d.CloudProvider)),
fmt.Sprintf("KOPS_FEATURE_FLAGS=%v", d.featureFlags()),
"KOPS_RUN_TOO_NEW_VERSION=1",
}...)
if d.CloudProvider == "aws" {
vars = append(vars, fmt.Sprintf("AWS_SHARED_CREDENTIALS_FILE=%v", os.Getenv("AWS_SHARED_CREDENTIALS_FILE")))
}
return vars
}
// featureFlags returns the kops feature flags to set
func (d *deployer) featureFlags() string {
return "+SpecOverrideFlag"
}
// defaultClusterName returns a kops cluster name to use when ClusterName is not set
func defaultClusterName(cloudProvider string) (string, error) {
jobName := os.Getenv("JOB_NAME")
buildID := os.Getenv("BUILD_ID")
if jobName == "" || buildID == "" {
return "", errors.New("JOB_NAME, and BUILD_ID env vars are required when --cluster-name is not set")
}
buildIDHash := md5.Sum([]byte(buildID))
jobHash := md5.Sum([]byte(jobName))
var suffix string
switch cloudProvider {
case "aws":
suffix = "test-cncf-aws.k8s.io"
default:
suffix = "k8s.local"
}
return fmt.Sprintf("e2e-%v-%v.%v", buildIDHash[:10], jobHash[:5], suffix), nil
}
// stateStore returns the kops state store to use
// defaulting to values used in prow jobs
func stateStore(cloudProvider string) string {
ss := os.Getenv("KOPS_STATE_STORE")
if ss == "" {
switch cloudProvider {
case "aws":
ss = "s3://k8s-kops-prow/"
case "gce":
ss = "gs://k8s-kops-gce/"
}
}
return ss
}

30
tests/e2e/kubetest2-kops/deployer/deployer.go
View File

@ -40,7 +40,18 @@ type deployer struct {
KopsRoot string `flag:"kops-root" desc:"Path to root of the kops repo. Used with --build."`
StageLocation string `flag:"stage-location" desc:"Storage location for kops artifacts. Only gs:// paths are supported."`
BuildOptions *builder.BuildOptions
ClusterName string `flag:"cluster-name" desc:"The FQDN to use for the cluster name"`
CloudProvider string `flag:"cloud-provider" desc:"Which cloud provider to use"`
Env []string `flag:"env" desc:"Additional env vars to set for kops commands in NAME=VALUE format"`
KopsBinaryPath string `flag:"kops-binary-path" desc:"The path to kops executable used for testing"`
StateStore string `flag:"-"`
SSHPrivateKeyPath string `flag:"ssh-private-key" desc:"The path to the private key used for SSH access to instances"`
SSHPublicKeyPath string `flag:"ssh-public-key" desc:"The path to the public key passed to the cloud provider"`
SSHUser []string `flag:"ssh-user" desc:"The SSH users to use for SSH access to instances"`
BuildOptions *builder.BuildOptions
}
// assert that New implements types.NewDeployer
@ -53,21 +64,6 @@ func (d *deployer) Provider() string {
return Name
}
func (d *deployer) Up() error {
klog.Warning("Up is not implemented")
return nil
}
func (d *deployer) IsUp() (bool, error) {
klog.Warning("IsUp is not implemented")
return true, nil
}
func (d *deployer) Down() error {
klog.Warning("Down is not implemented")
return nil
}
func (d *deployer) DumpClusterLogs() error {
klog.Warning("DumpClusterLogs is not implemented")
return nil
@ -93,7 +89,7 @@ func New(opts types.Options) (types.Deployer, *pflag.FlagSet) {
func bindFlags(d *deployer) *pflag.FlagSet {
flags, err := gpflag.Parse(d)
if err != nil {
klog.Fatalf("unable to generate flags from deployer")
klog.Fatalf("unable to generate flags from deployer: %v", err)
return nil
}
return flags

41
tests/e2e/kubetest2-kops/deployer/down.go Normal file
View File

@ -0,0 +1,41 @@
/*
Copyright 2020 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package deployer
import (
"strings"
"k8s.io/klog/v2"
"sigs.k8s.io/kubetest2/pkg/exec"
)
func (d *deployer) Down() error {
if err := d.init(); err != nil {
return err
}
args := []string{
d.KopsBinaryPath, "delete", "cluster",
"--name", d.ClusterName,
"--yes",
}
klog.Info(strings.Join(args, " "))
cmd := exec.Command(args[0], args[1:]...)
cmd.SetEnv(d.env()...)
exec.InheritOutput(cmd)
return cmd.Run()
}

93
tests/e2e/kubetest2-kops/deployer/up.go Normal file
View File

@ -0,0 +1,93 @@
/*
Copyright 2020 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package deployer
import (
"os"
osexec "os/exec"
"strings"
"k8s.io/klog/v2"
"k8s.io/kops/tests/e2e/kubetest2-kops/util"
"sigs.k8s.io/kubetest2/pkg/exec"
)
func (d *deployer) Up() error {
if err := d.init(); err != nil {
return err
}
publicIP, err := util.ExternalIPRange()
if err != nil {
return err
}
args := []string{
d.KopsBinaryPath, "create", "cluster",
"--name", d.ClusterName,
"--admin-access", publicIP,
"--cloud", d.CloudProvider,
"--master-count", "1",
"--master-size", "c5.large",
"--master-volume-size", "48",
"--node-count", "4",
"--node-volume-size", "48",
"--override", "cluster.spec.nodePortAccess=0.0.0.0/0",
"--ssh-public-key", d.SSHPublicKeyPath,
"--zones", "eu-west-2a",
"--yes",
}
klog.Info(strings.Join(args, " "))
cmd := exec.Command(args[0], args[1:]...)
cmd.SetEnv(d.env()...)
exec.InheritOutput(cmd)
return cmd.Run()
}
func (d *deployer) IsUp() (bool, error) {
args := []string{
d.KopsBinaryPath, "validate", "cluster",
"--name", d.ClusterName,
"--wait", "15m",
}
klog.Info(strings.Join(args, " "))
cmd := exec.Command(args[0], args[1:]...)
cmd.SetEnv(d.env()...)
exec.InheritOutput(cmd)
err := cmd.Run()
// `kops validate cluster` exits 2 if validation failed
if exitErr, ok := err.(*osexec.ExitError); ok && exitErr.ExitCode() == 2 {
return false, nil
}
return err == nil, err
}
// verifyUpFlags ensures fields are set for creation of the cluster
func (d *deployer) verifyUpFlags() error {
// These environment variables are defined by the "preset-aws-ssh" prow preset
// https://github.com/kubernetes/test-infra/blob/3d3b325c98b739b526ba5d93ce21c90a05e1f46d/config/prow/config.yaml#L653-L670
if d.SSHPrivateKeyPath == "" {
d.SSHPrivateKeyPath = os.Getenv("AWS_SSH_PRIVATE_KEY_FILE")
}
if d.SSHPublicKeyPath == "" {
d.SSHPublicKeyPath = os.Getenv("AWS_SSH_PUBLIC_KEY_FILE")
}
return nil
}

59
tests/e2e/kubetest2-kops/util/http.go Normal file
View File

@ -0,0 +1,59 @@
/*
Copyright 2020 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package util
import (
"fmt"
"io"
"net/http"
"k8s.io/klog/v2"
)
var httpTransport *http.Transport
func init() {
httpTransport = new(http.Transport)
httpTransport.Proxy = http.ProxyFromEnvironment
httpTransport.RegisterProtocol("file", http.NewFileTransport(http.Dir("/")))
}
// httpGETWithHeaders writes the response of an HTTP GET request
func httpGETWithHeaders(url string, headers map[string]string, writer io.Writer) error {
klog.Infof("curl %s", url)
c := &http.Client{Transport: httpTransport}
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return err
}
for k, v := range headers {
req.Header.Add(k, v)
}
r, err := c.Do(req)
if err != nil {
return err
}
defer r.Body.Close()
if r.StatusCode >= 400 {
return fmt.Errorf("%v returned %d", url, r.StatusCode)
}
_, err = io.Copy(writer, r.Body)
if err != nil {
return err
}
return nil
}

68
tests/e2e/kubetest2-kops/util/publicip.go Normal file
View File

@ -0,0 +1,68 @@
/*
Copyright 2020 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package util
import (
"bytes"
"fmt"
"log"
"net"
"strings"
"time"
)
const externalIPMetadataURL = "http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip"
var externalIPServiceURLs = []string{
"https://ip.jsb.workers.dev",
"https://v4.ifconfig.co",
}
// ExternalIPRange returns the CIDR block for the public IP
// in front of the kubetest2 client
func ExternalIPRange() (string, error) {
var b bytes.Buffer
err := httpGETWithHeaders(externalIPMetadataURL, map[string]string{"Metadata-Flavor": "Google"}, &b)
if err != nil {
// This often fails due to workload identity
log.Printf("failed to get external ip from metadata service: %v", err)
} else if ip := net.ParseIP(strings.TrimSpace(b.String())); ip != nil {
return ip.String() + "/32", nil
} else {
log.Printf("metadata service returned invalid ip %q", b.String())
}
for attempt := 0; attempt < 5; attempt++ {
for _, u := range externalIPServiceURLs {
b.Reset()
err = httpGETWithHeaders(u, nil, &b)
if err != nil {
// The external service may well be down
log.Printf("failed to get external ip from %s: %v", u, err)
} else if ip := net.ParseIP(strings.TrimSpace(b.String())); ip != nil {
return ip.String() + "/32", nil
} else {
log.Printf("service %s returned invalid ip %q", u, b.String())
}
}
time.Sleep(2 * time.Second)
}
return "", fmt.Errorf("external IP cannot be retrieved")
}