Merge pull request #10697 from rifelpet/kubetest-tf

Kubetest2 - terraform support

tests/e2e/kubetest2-kops/deployer/common.go

@@ -27,6 +27,7 @@ import (
 
 	"k8s.io/klog/v2"
 	"k8s.io/kops/tests/e2e/pkg/kops"
+	"k8s.io/kops/tests/e2e/pkg/target"
 )
 
 func (d *deployer) init() error {
@@ -71,6 +72,13 @@ func (d *deployer) initialize() error {
 	if d.SSHUser == "" {
 		d.SSHUser = os.Getenv("KUBE_SSH_USER")
 	}
+	if d.TerraformVersion != "" {
+		t, err := target.NewTerraform(d.TerraformVersion)
+		if err != nil {
+			return err
+		}
+		d.terraform = t
+	}
 	return nil
 }
 

tests/e2e/kubetest2-kops/deployer/create.go

@@ -46,6 +46,10 @@ func (d *deployer) replace() error {
 		"--admin",
 		"--name", d.ClusterName,
 	}
+	if d.terraform != nil {
+		args = append(args, "--target", "terraform", "--out", d.terraform.Dir())
+	}
+
 	klog.Info(strings.Join(args, " "))
 
 	cmd = exec.Command(args[0], args[1:]...)
@@ -56,5 +60,10 @@ func (d *deployer) replace() error {
 	if err != nil {
 		return err
 	}
+	if d.terraform != nil {
+		if err := d.terraform.InitApply(); err != nil {
+			return err
+		}
+	}
 	return nil
 }

tests/e2e/kubetest2-kops/deployer/deployer.go

@@ -25,6 +25,7 @@ import (
 	"github.com/spf13/pflag"
 	"k8s.io/klog/v2"
 	"k8s.io/kops/tests/e2e/kubetest2-kops/builder"
+	"k8s.io/kops/tests/e2e/pkg/target"
 
 	"sigs.k8s.io/kubetest2/pkg/types"
 )
@@ -59,6 +60,8 @@ type deployer struct {
 	SSHPublicKeyPath  string `flag:"ssh-public-key" desc:"The path to the public key passed to the cloud provider"`
 	SSHUser           string `flag:"ssh-user" desc:"The SSH user to use for SSH access to instances"`
 
+	TerraformVersion string `flag:"terraform-version" desc:"The version of terraform to use for applying the cluster"`
+
 	ArtifactsDir string `flag:"-"`
 
 	AdminAccess string `flag:"admin-access" desc:"The CIDR to restrict kubernetes API access"`
@@ -67,6 +70,7 @@ type deployer struct {
 
 	// manifestPath is the location of the rendered manifest based on TemplatePath
 	manifestPath string
+	terraform    *target.Terraform
 }
 
 // assert that New implements types.NewDeployer

tests/e2e/kubetest2-kops/deployer/down.go

@@ -31,6 +31,12 @@ func (d *deployer) Down() error {
 		klog.Warningf("Dumping cluster logs at the start of Down() failed: %s", err)
 	}
 
+	if d.terraform != nil {
+		if err := d.terraform.Destroy(); err != nil {
+			return err
+		}
+	}
+
 	args := []string{
 		d.KopsBinaryPath, "delete", "cluster",
 		"--name", d.ClusterName,

tests/e2e/kubetest2-kops/deployer/up.go

@@ -134,6 +134,7 @@ func (d *deployer) IsUp() (bool, error) {
 	args := []string{
 		d.KopsBinaryPath, "validate", "cluster",
 		"--name", d.ClusterName,
+		"--count", "10",
 		"--wait", "15m",
 	}
 	klog.Info(strings.Join(args, " "))

tests/e2e/pkg/target/terraform.go

@@ -0,0 +1,117 @@
+/*
+Copyright 2021 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package target
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"path/filepath"
+	"runtime"
+	"strings"
+
+	"k8s.io/klog/v2"
+	"k8s.io/kops/tests/e2e/pkg/util"
+	"sigs.k8s.io/kubetest2/pkg/exec"
+)
+
+// Terraform represents a set of terraform commands to be ran against a directory
+// containing a kops cluster's .tf output
+type Terraform struct {
+	dir           string
+	terraformPath string
+}
+
+// NewTerraform creates a new Terraform object
+func NewTerraform(version string) (*Terraform, error) {
+	var b bytes.Buffer
+	url := fmt.Sprintf("https://releases.hashicorp.com/terraform/%v/terraform_%v_%v_%v.zip", version, version, runtime.GOOS, runtime.GOARCH)
+
+	if err := util.HTTPGETWithHeaders(url, nil, &b); err != nil {
+		return nil, err
+	}
+	binaryDir, err := util.UnzipToTempDir(b.Bytes())
+	if err != nil {
+		return nil, err
+	}
+	tfDir, err := ioutil.TempDir("", "kops-terraform")
+	if err != nil {
+		return nil, err
+	}
+	t := &Terraform{
+		dir:           tfDir,
+		terraformPath: filepath.Join(binaryDir, "terraform"),
+	}
+	return t, nil
+}
+
+// Dir returns the directory containing the kops-generated .tf files
+func (t *Terraform) Dir() string {
+	return t.dir
+}
+
+// InitApply runs `terraform init` and `terraform apply` in the specified directory
+func (t *Terraform) InitApply() error {
+	args := []string{
+		t.terraformPath, "init",
+	}
+	klog.Info(strings.Join(args, " "))
+
+	cmd := exec.Command(args[0], args[1:]...)
+	cmd.SetDir(t.dir)
+
+	exec.InheritOutput(cmd)
+	err := cmd.Run()
+	if err != nil {
+		return err
+	}
+
+	args = []string{
+		t.terraformPath, "apply",
+		"-auto-approve",
+	}
+	klog.Info(strings.Join(args, " "))
+
+	cmd = exec.Command(args[0], args[1:]...)
+	cmd.SetDir(t.dir)
+
+	exec.InheritOutput(cmd)
+	err = cmd.Run()
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// Destroy runs `terraform destroy` in the specified directory
+func (t *Terraform) Destroy() error {
+	args := []string{
+		t.terraformPath, "destroy",
+		"-auto-approve",
+	}
+	klog.Info(strings.Join(args, " "))
+
+	cmd := exec.Command(args[0], args[1:]...)
+	cmd.SetDir(t.dir)
+
+	exec.InheritOutput(cmd)
+	err := cmd.Run()
+	if err != nil {
+		return err
+	}
+	return nil
+}

tests/e2e/pkg/util/zip.go

@@ -0,0 +1,76 @@
+/*
+Copyright 2021 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"archive/zip"
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+// UnzipToTempDir will decompress the provided bytes into a temporary directory that is returned
+func UnzipToTempDir(data []byte) (string, error) {
+	reader, err := zip.NewReader(bytes.NewReader(data), int64(len(data)))
+	if err != nil {
+		return "", err
+	}
+	dir, err := ioutil.TempDir("", "")
+	if err != nil {
+		return "", err
+	}
+
+	for _, r := range reader.File {
+		fileNamePath := filepath.Join(dir, r.Name)
+		if !strings.HasPrefix(fileNamePath, filepath.Clean(dir)+string(os.PathSeparator)) {
+			return "", fmt.Errorf("invalid file path: %v", fileNamePath)
+		}
+
+		if r.FileInfo().IsDir() {
+			os.MkdirAll(fileNamePath, os.ModePerm)
+			continue
+		}
+
+		if err = os.MkdirAll(filepath.Dir(fileNamePath), os.ModePerm); err != nil {
+			return "", err
+		}
+
+		output, err := os.OpenFile(fileNamePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, r.Mode())
+		if err != nil {
+			return "", err
+		}
+
+		fileReader, err := r.Open()
+		if err != nil {
+			return "", err
+		}
+
+		_, err = io.Copy(output, fileReader)
+
+		output.Close()
+		fileReader.Close()
+
+		if err != nil {
+			return "", err
+		}
+	}
+	return dir, nil
+}