kubetest2 - Add manifest template support

I'm updating the test-e2e make target as I go to make the presubmit job use it but soon I'll work on how we'll actually configure and invoke kubetest2 from our variety of jobs

Makefile

@@ -198,6 +198,7 @@ test-e2e:
 		--cloud-provider=aws \
 		--kops-binary-path=/home/prow/go/src/k8s.io/kops/bazel-bin/cmd/kops/linux-amd64/kops \
 		--kubernetes-version=v1.19.4 \
+		--template-path=tests/e2e/templates/simple.yaml.tmpl \
 		--test=kops \
 		-- \
 		--test-package-version=v1.19.4 \

tests/e2e/go.mod

@@ -5,6 +5,7 @@ go 1.15
 require (
 	github.com/octago/sflags v0.2.0
 	github.com/spf13/pflag v1.0.5
+	gopkg.in/yaml.v2 v2.3.0
 	k8s.io/klog/v2 v2.4.0
 	sigs.k8s.io/kubetest2 v0.0.0-20201130212850-d9dad7c8699c
 )

tests/e2e/go.sum

@@ -1493,6 +1493,7 @@ gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v3 v3.0.0-20190709130402-674ba3eaed22/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=

tests/e2e/kubetest2-kops/deployer/common.go

@@ -85,6 +85,10 @@ func (d *deployer) verifyKopsFlags() error {
 		return errors.New("unsupported --cloud-provider value")
 	}
 
+	if d.StateStore == "" {
+		d.StateStore = stateStore(d.CloudProvider)
+	}
+
 	return nil
 }
 
@@ -94,7 +98,7 @@ func (d *deployer) env() []string {
 	vars = append(vars, []string{
 		fmt.Sprintf("PATH=%v", os.Getenv("PATH")),
 		fmt.Sprintf("HOME=%v", os.Getenv("HOME")),
-		fmt.Sprintf("KOPS_STATE_STORE=%v", stateStore(d.CloudProvider)),
+		fmt.Sprintf("KOPS_STATE_STORE=%v", d.StateStore),
 		fmt.Sprintf("KOPS_FEATURE_FLAGS=%v", d.featureFlags()),
 		"KOPS_RUN_TOO_NEW_VERSION=1",
 	}...)

tests/e2e/kubetest2-kops/deployer/deployer.go

@@ -47,6 +47,8 @@ type deployer struct {
 	KopsBinaryPath string   `flag:"kops-binary-path" desc:"The path to kops executable used for testing"`
 	StateStore     string   `flag:"-"`
 
+	TemplatePath string `flag:"template-path" desc:"The path to the manifest template used for cluster creation"`
+
 	KubernetesVersion string `flag:"kubernetes-version" desc:"The kubernetes version to use in the cluster"`
 
 	SSHPrivateKeyPath string   `flag:"ssh-private-key" desc:"The path to the private key used for SSH access to instances"`
@@ -58,6 +60,9 @@ type deployer struct {
 	AdminAccess string `flag:"admin-access" desc:"The CIDR to restrict kubernetes API access"`
 
 	BuildOptions *builder.BuildOptions
+
+	// manifestPath is the location of the rendered manifest based on TemplatePath
+	manifestPath string
 }
 
 // assert that New implements types.NewDeployer

tests/e2e/kubetest2-kops/deployer/replace.go

@@ -0,0 +1,76 @@
+/*
+Copyright 2021 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package deployer
+
+import (
+	"strings"
+
+	"k8s.io/klog/v2"
+	"sigs.k8s.io/kubetest2/pkg/exec"
+)
+
+// replace performs a `kops replace` followed by `kops update cluster --yes`
+func (d *deployer) replace() error {
+	args := []string{
+		d.KopsBinaryPath, "replace",
+		"--filename", d.manifestPath,
+		"--force",
+		"--name", d.ClusterName,
+	}
+	klog.Info(strings.Join(args, " "))
+
+	cmd := exec.Command(args[0], args[1:]...)
+	cmd.SetEnv(d.env()...)
+
+	exec.InheritOutput(cmd)
+	err := cmd.Run()
+	if err != nil {
+		return err
+	}
+
+	args = []string{
+		d.KopsBinaryPath, "create", "secret", "sshpublickey",
+		"admin",
+		"-i", d.SSHPublicKeyPath,
+		"--name", d.ClusterName,
+	}
+	klog.Info(strings.Join(args, " "))
+	cmd = exec.Command(args[0], args[1:]...)
+	cmd.SetEnv(d.env()...)
+
+	exec.InheritOutput(cmd)
+	err = cmd.Run()
+	if err != nil {
+		return err
+	}
+
+	args = []string{
+		d.KopsBinaryPath, "update", "cluster", "--yes",
+		"--name", d.ClusterName,
+	}
+	klog.Info(strings.Join(args, " "))
+
+	cmd = exec.Command(args[0], args[1:]...)
+	cmd.SetEnv(d.env()...)
+
+	exec.InheritOutput(cmd)
+	err = cmd.Run()
+	if err != nil {
+		return err
+	}
+	return nil
+}

tests/e2e/kubetest2-kops/deployer/template.go

@@ -0,0 +1,75 @@
+/*
+Copyright 2021 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package deployer
+
+import (
+	"io/ioutil"
+	"path"
+	"strings"
+
+	"gopkg.in/yaml.v2"
+	"k8s.io/klog/v2"
+	"sigs.k8s.io/kubetest2/pkg/exec"
+)
+
+// renderTemplate will render the manifest template with the provided values,
+// setting the deployer's manifestPath
+func (d *deployer) renderTemplate(values map[string]interface{}) error {
+	dir, err := ioutil.TempDir("", "kops-template")
+	if err != nil {
+		return err
+	}
+
+	valuesBytes, err := yaml.Marshal(values)
+	if err != nil {
+		return err
+	}
+	valuesPath := path.Join(dir, "values.yaml")
+	ioutil.WriteFile(valuesPath, valuesBytes, 0644)
+
+	manifestPath := path.Join(dir, "manifest.yaml")
+	d.manifestPath = manifestPath
+
+	args := []string{
+		d.KopsBinaryPath, "toolbox", "template",
+		"--template", d.TemplatePath,
+		"--output", manifestPath,
+		"--values", valuesPath,
+	}
+	klog.Info(strings.Join(args, " "))
+
+	cmd := exec.Command(args[0], args[1:]...)
+	cmd.SetEnv(d.env()...)
+
+	exec.InheritOutput(cmd)
+	err = cmd.Run()
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (d *deployer) templateValues(zones []string, publicIP string) map[string]interface{} {
+	return map[string]interface{}{
+		"cloudProvider":     d.CloudProvider,
+		"clusterName":       d.ClusterName,
+		"kubernetesVersion": d.KubernetesVersion,
+		"publicIP":          publicIP,
+		"stateStore":        d.StateStore,
+		"zones":             zones,
+	}
+}

tests/e2e/kubetest2-kops/deployer/up.go

@@ -17,6 +17,7 @@ limitations under the License.
 package deployer
 
 import (
+	"fmt"
 	"os"
 	osexec "os/exec"
 	"strings"
@@ -43,6 +44,38 @@ func (d *deployer) Up() error {
 		adminAccess = publicIP
 	}
 
+	zones, err := d.zones()
+	if err != nil {
+		return err
+	}
+
+	if d.TemplatePath != "" {
+		values := d.templateValues(zones, adminAccess)
+		if err := d.renderTemplate(values); err != nil {
+			return err
+		}
+		if err := d.replace(); err != nil {
+			return err
+		}
+	} else {
+		err := d.createCluster(zones, adminAccess)
+		if err != nil {
+			return err
+		}
+	}
+	isUp, err := d.IsUp()
+	if err != nil {
+		return err
+	} else if isUp {
+		klog.V(1).Infof("cluster reported as up")
+	} else {
+		klog.Errorf("cluster reported as down")
+	}
+	return nil
+}
+
+func (d *deployer) createCluster(zones []string, adminAccess string) error {
+
 	args := []string{
 		d.KopsBinaryPath, "create", "cluster",
 		"--name", d.ClusterName,
@@ -83,20 +116,10 @@ func (d *deployer) Up() error {
 	cmd.SetEnv(d.env()...)
 
 	exec.InheritOutput(cmd)
-	err = cmd.Run()
+	err := cmd.Run()
 	if err != nil {
 		return err
 	}
-
-	isUp, err := d.IsUp()
-	if err != nil {
-		return err
-	} else if isUp {
-		klog.V(1).Infof("cluster reported as up")
-	} else {
-		klog.Errorf("cluster reported as down")
-	}
-
 	return nil
 }
 
@@ -133,3 +156,13 @@ func (d *deployer) verifyUpFlags() error {
 
 	return nil
 }
+
+func (d *deployer) zones() ([]string, error) {
+	switch d.CloudProvider {
+	case "aws":
+		return aws.RandomZones(1)
+	case "gce":
+		return gce.RandomZones(1)
+	}
+	return nil, fmt.Errorf("unsupported CloudProvider: %v", d.CloudProvider)
+}

tests/e2e/templates/simple.yaml.tmpl

@@ -0,0 +1,79 @@
+{{$zone := index .zones 0}}
+apiVersion: kops.k8s.io/v1alpha2
+kind: Cluster
+metadata:
+  name: {{.clusterName}}
+spec:
+  kubernetesApiAccess:
+  - {{.publicIP}}
+  channel: stable
+  cloudProvider: {{.cloudProvider}}
+  configBase: "{{.stateStore}}/{{.clusterName}}"
+  etcdClusters:
+  - etcdMembers:
+    - instanceGroup: master-{{$zone}}
+      name: {{$zone}}
+    name: main
+  - etcdMembers:
+    - instanceGroup: master-{{$zone}}
+      name: {{$zone}}
+    name: events
+  iam: {}
+  kubelet:
+    anonymousAuth: false
+  kubernetesVersion: {{.kubernetesVersion}}
+  masterInternalName: api.internal.{{.clusterName}}
+  masterPublicName: api.{{.clusterName}}
+  networkCIDR: 172.20.0.0/16
+  networking:
+    kubenet: {}
+  nodePortAccess:
+    - 0.0.0.0/0
+  nonMasqueradeCIDR: 100.64.0.0/10
+  sshAccess:
+    - {{.publicIP}}
+  topology:
+    masters: public
+    nodes: public
+  subnets:
+  - cidr: 172.20.32.0/19
+    name: {{$zone}}
+    type: Public
+    zone: {{$zone}}
+
+---
+
+apiVersion: kops.k8s.io/v1alpha2
+kind: InstanceGroup
+metadata:
+  name: nodes-{{$zone}}
+  labels:
+    kops.k8s.io/cluster: {{.clusterName}}
+spec:
+  associatePublicIp: true
+  image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20201112.1
+  machineType: t3.medium
+  maxSize: 4
+  minSize: 4
+  role: Node
+  subnets:
+  - {{$zone}}
+
+---
+
+apiVersion: kops.k8s.io/v1alpha2
+kind: InstanceGroup
+metadata:
+  name: master-{{$zone}}
+  labels:
+    kops.k8s.io/cluster: {{.clusterName}}
+spec:
+  associatePublicIp: true
+  image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20201112.1
+  machineType: c5.large
+  maxSize: 1
+  minSize: 1
+  role: Master
+  subnets:
+  - {{$zone}}
+