Add prow-cluster-operation package (#788)

* Add prow-cluster-operation package

* Remove binary committed by accident

testutils/clustermanager/prow-cluster-operation/README.md

@@ -0,0 +1,53 @@
+## prow-cluster-operation
+
+prow-cluster-operation is a tool for creating, deleting, getting a GKE
+cluster
+
+## Prerequisite
+
+- `GOOGLE_APPLICATION_CREDENTIALS` set
+
+## Usage
+
+This tool can be invoked from command line with following parameters:
+
+- `--min-nodes`: minumum number of nodes, default 1
+- `--max-nodes`: maximum number of nodes, default 3
+- `--node-type`: GCE node type, default "n1-standard-4"
+- `--region`: GKE region, default "us-central1"
+- `--zone`: GKE zone, default empty
+- `--project`: GCP project, default empty
+- `--name`: cluster name, default empty
+- `--backup-regions`: backup regions to be used if cluster creation in primary
+  region failed, comma separated list, default "us-west1,us-east1"
+- `--addons`: GKE addons, comma separated list, default empty
+
+## Flow
+
+### Create
+
+1. Acquiring cluster if kubeconfig already points to it
+1. Get GCP project name if not provided as a parameter:
+    - [In Prow] Acquire from Boskos
+    - [Not in Prow] Read from gcloud config
+
+    Failed obtaining project name will fail the tool
+1. Get default cluster name if not provided as a parameter
+1. Delete cluster if cluster with same name and location already exists in GKE
+1. Create cluster
+1. Write cluster metadata to `${ARTIFACT}/metadata.json`
+
+### Delete
+
+1. Acquiring cluster if kubeconfig already points to it
+1. If cluster name is defined then getting cluster by its name
+1. If no cluster is found from previous step then it fails
+1. Delete:
+    - [In Prow] Release Boskos project
+    - [Not in Prow] Delete cluster
+
+### Get
+
+1. Acquiring cluster if kubeconfig already points to it
+1. If cluster name is defined then getting cluster by its name
+1. If no cluster is found from previous step then it fails

testutils/clustermanager/prow-cluster-operation/actions/create.go

@@ -0,0 +1,108 @@
+/*
+Copyright 2019 The Knative Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package actions
+
+import (
+	"log"
+	"strconv"
+
+	container "google.golang.org/api/container/v1beta1"
+	"knative.dev/pkg/test/gke"
+	clm "knative.dev/pkg/testutils/clustermanager/e2e-tests"
+	"knative.dev/pkg/testutils/clustermanager/e2e-tests/common"
+	"knative.dev/pkg/testutils/clustermanager/prow-cluster-operation/options"
+	"knative.dev/pkg/testutils/metahelper/client"
+)
+
+const (
+	// Keys to be written into metadata.json
+	e2eRegionKey      = "E2E:Region"
+	e2eZoneKey        = "E2E:Zone"
+	clusterNameKey    = "E2E:Machine"
+	clusterVersionKey = "E2E:Version"
+	minNodesKey       = "E2E:MinNodes"
+	maxNodesKey       = "E2E:MaxNodes"
+	projectKey        = "E2E:Project"
+)
+
+func writeMetaData(cluster *container.Cluster, project string) {
+	// Set up metadata client for saving metadata
+	c, err := client.NewClient("")
+	if err != nil {
+		log.Fatal(err)
+	}
+	log.Printf("Writing metadata to: %q", c.Path)
+	// Get minNodes and maxNodes counts from default-pool, this is
+	// usually the case in tests in Prow
+	var minNodes, maxNodes string
+	for _, np := range cluster.NodePools {
+		if np.Name == "default-pool" {
+			minNodes = strconv.FormatInt(np.InitialNodeCount, 10)
+			// maxNodes is equal to minNodes if autoscaling isn't on
+			maxNodes = minNodes
+			if np.Autoscaling != nil {
+				minNodes = strconv.FormatInt(np.Autoscaling.MinNodeCount, 10)
+				maxNodes = strconv.FormatInt(np.Autoscaling.MaxNodeCount, 10)
+			} else {
+				log.Printf("DEBUG: nodepool is default-pool but autoscaling is not on: '%+v'", np)
+			}
+			break
+		}
+	}
+
+	e2eRegion, e2eZone := gke.RegionZoneFromLoc(cluster.Location)
+	for key, val := range map[string]string{
+		e2eRegionKey:      e2eRegion,
+		e2eZoneKey:        e2eZone,
+		clusterNameKey:    cluster.Name,
+		clusterVersionKey: cluster.InitialClusterVersion,
+		minNodesKey:       minNodes,
+		maxNodesKey:       maxNodes,
+		projectKey:        project,
+	} {
+		if err = c.Set(key, val); err != nil {
+			log.Fatalf("Failed saving metadata %q:%q: '%v'", key, val, err)
+		}
+	}
+	log.Println("Done writing metadata")
+}
+
+func Create(o *options.RequestWrapper) {
+	o.Prep()
+
+	gkeClient := clm.GKEClient{}
+	clusterOps := gkeClient.Setup(o.Request)
+	gkeOps := clusterOps.(*clm.GKECluster)
+	if err := gkeOps.Acquire(); err != nil || gkeOps.Cluster == nil {
+		log.Fatalf("failed acquiring GKE cluster: '%v'", err)
+	}
+
+	// At this point we should have a cluster ready to run test. Need to save
+	// metadata so that following flow can understand the context of cluster, as
+	// well as for Prow usage later
+	writeMetaData(gkeOps.Cluster, gkeOps.Project)
+
+	// set up kube config points to cluster
+	// TODO(chaodaiG): this probably should also be part of clustermanager lib
+	if out, err := common.StandardExec("gcloud", "beta", "container", "clusters", "get-credentials",
+		gkeOps.Cluster.Name, "--region", gkeOps.Cluster.Location, "--project", gkeOps.Project); err != nil {
+		log.Fatalf("Failed connecting to cluster: %q, '%v'", out, err)
+	}
+	if out, err := common.StandardExec("gcloud", "config", "set", "project", gkeOps.Project); err != nil {
+		log.Fatalf("Failed setting gcloud: %q, '%v'", out, err)
+	}
+}

testutils/clustermanager/prow-cluster-operation/actions/delete.go

@@ -0,0 +1,50 @@
+/*
+Copyright 2019 The Knative Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package actions
+
+import (
+	"log"
+
+	clm "knative.dev/pkg/testutils/clustermanager/e2e-tests"
+	"knative.dev/pkg/testutils/clustermanager/prow-cluster-operation/options"
+)
+
+func Delete(o *options.RequestWrapper) {
+	o.Request.NeedsCleanup = true
+	o.Request.SkipCreation = true
+
+	gkeClient := clm.GKEClient{}
+	clusterOps := gkeClient.Setup(o.Request)
+	gkeOps := clusterOps.(*clm.GKECluster)
+	if err := gkeOps.Acquire(); err != nil || gkeOps.Cluster == nil {
+		log.Fatalf("Failed identifying cluster for cleanup: '%v'", err)
+	}
+	log.Printf("Identified project %q and cluster %q for removal", gkeOps.Project, gkeOps.Cluster.Name)
+	var err error
+	if err = gkeOps.Delete(); err != nil {
+		log.Fatalf("Failed deleting cluster: '%v'", err)
+	}
+	// TODO: uncomment the lines below when previous Delete command becomes
+	// async operation
+	// // Unset context with best effort. The first command only unsets current
+	// // context, but doesn't delete the entry from kubeconfig, and should return it's
+	// // context if succeeded, which can be used by the second command to
+	// // delete it from kubeconfig
+	// if out, err := common.StandardExec("kubectl", "config", "unset", "current-context"); err != nil {
+	// 	common.StandardExec("kubectl", "config", "unset", "contexts."+string(out))
+	// }
+}

testutils/clustermanager/prow-cluster-operation/actions/get.go

@@ -0,0 +1,29 @@
+/*
+Copyright 2019 The Knative Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package actions
+
+import (
+	"knative.dev/pkg/testutils/clustermanager/prow-cluster-operation/options"
+)
+
+func Get(o *options.RequestWrapper) {
+	o.Prep()
+	o.Request.SkipCreation = true
+	// Reuse `Create` for getting operation, so that we can reuse the same logic
+	// such as protected project/cluster etc.
+	Create(o)
+}

testutils/clustermanager/prow-cluster-operation/main.go

@@ -0,0 +1,53 @@
+/*
+Copyright 2019 The Knative Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"flag"
+	"log"
+
+	"knative.dev/pkg/testutils/clustermanager/prow-cluster-operation/actions"
+	"knative.dev/pkg/testutils/clustermanager/prow-cluster-operation/options"
+)
+
+var (
+	create bool
+	delete bool
+	get    bool
+)
+
+func main() {
+	flag.BoolVar(&create, "create", false, "Create cluster")
+	flag.BoolVar(&delete, "delete", false, "Delete cluster")
+	flag.BoolVar(&get, "get", false, "Get existing cluster from kubeconfig or gcloud")
+	o := options.NewRequestWrapper()
+	flag.Parse()
+
+	if (create && delete) || (create && get) || (delete && get) {
+		log.Fatal("--create, --delete, --get are mutually exclusive")
+	}
+	switch {
+	case create:
+		actions.Create(o)
+	case delete:
+		actions.Delete(o)
+	case get:
+		actions.Get(o)
+	default:
+		log.Fatal("Must pass one of --create, --delete, --get")
+	}
+}

testutils/clustermanager/prow-cluster-operation/options/options.go

@@ -0,0 +1,61 @@
+/*
+Copyright 2019 The Knative Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package options
+
+import (
+	"flag"
+	"strings"
+
+	clm "knative.dev/pkg/testutils/clustermanager/e2e-tests"
+)
+
+type RequestWrapper struct {
+	Request          clm.GKERequest
+	BackupRegionsStr string
+	AddonsStr        string
+	NoWait           bool
+}
+
+func NewRequestWrapper() *RequestWrapper {
+	rw := &RequestWrapper{
+		Request: clm.GKERequest{},
+	}
+	rw.addOptions()
+	return rw
+}
+
+func (rw *RequestWrapper) Prep() {
+	if rw.BackupRegionsStr != "" {
+		rw.Request.BackupRegions = strings.Split(rw.BackupRegionsStr, ",")
+	}
+	if rw.AddonsStr != "" {
+		rw.Request.Addons = strings.Split(rw.AddonsStr, ",")
+	}
+}
+
+func (rw *RequestWrapper) addOptions() {
+	flag.Int64Var(&rw.Request.MinNodes, "min-nodes", 0, "minimal number of nodes")
+	flag.Int64Var(&rw.Request.MaxNodes, "max-nodes", 0, "maximal number of nodes")
+	flag.StringVar(&rw.Request.NodeType, "node-type", "", "node type")
+	flag.StringVar(&rw.Request.Region, "region", "", "GCP region")
+	flag.StringVar(&rw.Request.Zone, "zone", "", "GCP zone")
+	flag.StringVar(&rw.Request.Project, "project", "", "GCP project")
+	flag.StringVar(&rw.Request.ClusterName, "name", "", "cluster name")
+	flag.StringVar(&rw.BackupRegionsStr, "backup-regions", "", "GCP regions as backup, separated by comma")
+	flag.StringVar(&rw.AddonsStr, "addons", "", "addons to be added, separated by comma")
+	flag.BoolVar(&rw.Request.SkipCreation, "skip-creation", false, "should skip creation or not")
+}