From d7f2dc2e82522bfd4bbcb41b26fda685153d9eb1 Mon Sep 17 00:00:00 2001 From: mattmoor-sockpuppet Date: Thu, 22 Aug 2019 07:13:35 -0700 Subject: [PATCH] Auto-update dependencies (#79) Produced via: `dep ensure -update knative.dev/test-infra knative.dev/pkg` /assign mattmoor --- Gopkg.lock | 8 +- vendor/knative.dev/pkg/Gopkg.lock | 41 +++- vendor/knative.dev/pkg/Gopkg.toml | 2 +- vendor/knative.dev/pkg/test/mako/README.md | 7 + vendor/knative.dev/pkg/test/mako/analyzer.go | 35 +++ vendor/knative.dev/pkg/test/mako/benchmark.go | 71 ++++++ vendor/knative.dev/pkg/test/mako/config.go | 60 +++++ .../knative.dev/pkg/test/mako/environment.go | 52 ++++ vendor/knative.dev/pkg/test/mako/sidecar.go | 96 ++++++++ .../pkg/test/mako/testdata/config-mako.yaml | 47 ++++ vendor/knative.dev/pkg/test/mako/time.go | 26 ++ .../pkg/testutils/clustermanager/gke.go | 222 ++++++++++++++++-- .../pkg/testutils/clustermanager/util.go | 7 + .../test-infra/scripts/e2e-tests.sh | 6 +- .../test-infra/scripts/presubmit-tests.sh | 15 +- 15 files changed, 649 insertions(+), 46 deletions(-) create mode 100644 vendor/knative.dev/pkg/test/mako/README.md create mode 100644 vendor/knative.dev/pkg/test/mako/analyzer.go create mode 100644 vendor/knative.dev/pkg/test/mako/benchmark.go create mode 100644 vendor/knative.dev/pkg/test/mako/config.go create mode 100644 vendor/knative.dev/pkg/test/mako/environment.go create mode 100644 vendor/knative.dev/pkg/test/mako/sidecar.go create mode 100644 vendor/knative.dev/pkg/test/mako/testdata/config-mako.yaml create mode 100644 vendor/knative.dev/pkg/test/mako/time.go diff --git a/Gopkg.lock b/Gopkg.lock index 45426df9..f6da0727 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -927,7 +927,7 @@ [[projects]] branch = "master" - digest = "1:b9a2a90a672cca14c7353eff529b8375db18878fd03fff30a63cb5500decdc08" + digest = "1:5dddb45857ff59292216934b6f39415befab65246afac26ba14976e3ec11a343" name = "knative.dev/pkg" packages = [ "apis", @@ -946,18 +946,18 @@ "metrics/metricskey", ] pruneopts = "T" - revision = "c11c79155fa4be834735d1d0cbcafad88b16be69" + revision = "9f6e5334c25aeebd862b19cb6715f9fe5aa56467" [[projects]] branch = "master" - digest = "1:305af75fc194c059ae710a75276ca8f1870c959214b18ed33f7ef8671995947f" + digest = "1:a68a49f889453ee0abab8d32ceaa19c12a1e86c7919e9cecb3d5ceac6aac199c" name = "knative.dev/test-infra" packages = [ "scripts", "tools/dep-collector", ] pruneopts = "UT" - revision = "f3fe0bcc30693df8e3c9616358b16045efb9ed10" + revision = "d2746f8b9470e8c8452a997e95190aba5320678a" [solve-meta] analyzer-name = "dep" diff --git a/vendor/knative.dev/pkg/Gopkg.lock b/vendor/knative.dev/pkg/Gopkg.lock index 3c7e41cd..b37b2eb4 100644 --- a/vendor/knative.dev/pkg/Gopkg.lock +++ b/vendor/knative.dev/pkg/Gopkg.lock @@ -223,6 +223,23 @@ pruneopts = "NUT" revision = "c3068f13fcc3961fd05f96f13c8250e350db4209" +[[projects]] + digest = "1:3efb665a5beaa0266ff287cdb58dff8a966631000e9f8ad8d832a288b90ca247" + name = "github.com/google/mako" + packages = [ + "clients/proto/analyzers/threshold_analyzer_go_proto", + "clients/proto/analyzers/utest_analyzer_go_proto", + "clients/proto/analyzers/window_deviation_go_proto", + "helpers/go/quickstore", + "helpers/proto/quickstore/quickstore_go_proto", + "internal/go/common", + "internal/quickstore_microservice/proto/quickstore_go_proto", + "spec/proto/mako_go_proto", + ] + pruneopts = "NUT" + revision = "d56a6e811df75f8e4d6e7c256d25acf19ef16d03" + version = "v0.0.0-rc.4" + [[projects]] digest = "1:ab3ec1fe3e39bac4b3ab63390767766622be35b7cab03f47f787f9ec60522a53" name = "github.com/google/uuid" @@ -699,9 +716,13 @@ [[projects]] branch = "master" - digest = "1:9b9245bd124d95af7072487cd1e5861174b859ebc31cbe9fbab3b88456701485" + digest = "1:bcfc1adc3202f2ff120bc4686c613c8f34af0f9e3de49b55d3bbecd70dc6651b" name = "google.golang.org/api" packages = [ + "container/v1", + "gensupport", + "googleapi", + "googleapi/internal/uritemplates", "googleapi/transport", "internal", "iterator", @@ -1184,14 +1205,11 @@ [[projects]] branch = "master" - digest = "1:013d8728ceab59db2a90db464dc0d3c1f677bdf95f974fed26fe6094b5504cf2" + digest = "1:df1c1cf6d2c9ca0781705ef095cb15e06f777cddb493775aad41b72f79867d7a" name = "k8s.io/test-infra" packages = [ - "boskos", "boskos/client", "boskos/common", - "boskos/crds", - "boskos/ranch", "boskos/storage", ] pruneopts = "NUT" @@ -1199,14 +1217,14 @@ [[projects]] branch = "master" - digest = "1:39bb1014c454fbf9d8f4eab62a44fcf8d3ade774e2b42015e533e5d9f3fe23d9" + digest = "1:305af75fc194c059ae710a75276ca8f1870c959214b18ed33f7ef8671995947f" name = "knative.dev/test-infra" packages = [ "scripts", "tools/dep-collector", ] pruneopts = "UT" - revision = "d65201fadb82dbdc5713f9ba9a1313150c7aa9b5" + revision = "f3fe0bcc30693df8e3c9616358b16045efb9ed10" [[projects]] digest = "1:8730e0150dfb2b7e173890c8b9868e7a273082ef8e39f4940e3506a481cf895c" @@ -1229,8 +1247,13 @@ "github.com/evanphx/json-patch", "github.com/ghodss/yaml", "github.com/golang/glog", + "github.com/golang/protobuf/proto", "github.com/google/go-cmp/cmp", "github.com/google/go-cmp/cmp/cmpopts", + "github.com/google/mako/clients/proto/analyzers/threshold_analyzer_go_proto", + "github.com/google/mako/helpers/go/quickstore", + "github.com/google/mako/helpers/proto/quickstore/quickstore_go_proto", + "github.com/google/mako/spec/proto/mako_go_proto", "github.com/google/uuid", "github.com/gorilla/websocket", "github.com/markbates/inflect", @@ -1253,7 +1276,10 @@ "go.uber.org/zap", "go.uber.org/zap/zapcore", "go.uber.org/zap/zaptest", + "golang.org/x/net/context", + "golang.org/x/oauth2/google", "golang.org/x/sync/errgroup", + "google.golang.org/api/container/v1", "gopkg.in/yaml.v2", "k8s.io/api/admission/v1beta1", "k8s.io/api/admissionregistration/v1beta1", @@ -1323,7 +1349,6 @@ "k8s.io/gengo/namer", "k8s.io/gengo/types", "k8s.io/klog", - "k8s.io/test-infra/boskos", "k8s.io/test-infra/boskos/client", "k8s.io/test-infra/boskos/common", "knative.dev/test-infra/scripts", diff --git a/vendor/knative.dev/pkg/Gopkg.toml b/vendor/knative.dev/pkg/Gopkg.toml index fe631ab0..8cd92c45 100644 --- a/vendor/knative.dev/pkg/Gopkg.toml +++ b/vendor/knative.dev/pkg/Gopkg.toml @@ -8,7 +8,7 @@ required = [ "k8s.io/code-generator/cmd/client-gen", "k8s.io/code-generator/cmd/lister-gen", "k8s.io/code-generator/cmd/informer-gen", - "k8s.io/test-infra/boskos", + "google.golang.org/api/container/v1", "github.com/evanphx/json-patch", "knative.dev/test-infra/scripts", "knative.dev/test-infra/tools/dep-collector", diff --git a/vendor/knative.dev/pkg/test/mako/README.md b/vendor/knative.dev/pkg/test/mako/README.md new file mode 100644 index 00000000..1bc749fe --- /dev/null +++ b/vendor/knative.dev/pkg/test/mako/README.md @@ -0,0 +1,7 @@ +# Mako + +[Mako](https://github.com/google/mako) is an open source project for performance testing in Knative. +It offers capacities like data storage, charting, statistical aggregation and automated regression analysis. + +This folder contains common code that can be used by all Knative projects, with which we can follow the +same process to set up Mako and collaborate. diff --git a/vendor/knative.dev/pkg/test/mako/analyzer.go b/vendor/knative.dev/pkg/test/mako/analyzer.go new file mode 100644 index 00000000..d581543b --- /dev/null +++ b/vendor/knative.dev/pkg/test/mako/analyzer.go @@ -0,0 +1,35 @@ +/* +Copyright 2019 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mako + +import ( + "github.com/golang/protobuf/proto" + tpb "github.com/google/mako/clients/proto/analyzers/threshold_analyzer_go_proto" + mpb "github.com/google/mako/spec/proto/mako_go_proto" +) + +// NewCrossRunConfig returns a config that can be used in ThresholdAnalyzer. +// By using it, the Analyzer will only fail if there are xx continuous runs that cross the threshold. +func NewCrossRunConfig(runCount int32, tags ...string) *tpb.CrossRunConfig { + return &tpb.CrossRunConfig{ + RunInfoQueryList: []*mpb.RunInfoQuery{{ + Limit: proto.Int32(runCount), + Tags: tags, + }}, + MinRunCount: proto.Int32(runCount), + } +} diff --git a/vendor/knative.dev/pkg/test/mako/benchmark.go b/vendor/knative.dev/pkg/test/mako/benchmark.go new file mode 100644 index 00000000..da143e1e --- /dev/null +++ b/vendor/knative.dev/pkg/test/mako/benchmark.go @@ -0,0 +1,71 @@ +/* +Copyright 2019 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mako + +import ( + "fmt" + "io/ioutil" + "log" + "os" + "path/filepath" + + "github.com/golang/protobuf/proto" + mpb "github.com/google/mako/spec/proto/mako_go_proto" +) + +const koDataPathEnvName = "KO_DATA_PATH" + +// MustGetBenchmark wraps getBenchmark in log.Fatalf +func MustGetBenchmark() *string { + b, err := getBenchmark() + if err != nil { + log.Fatalf("unable to determine benchmark_key: %v", err) + } + return b +} + +// getBenchmark fetches the appropriate benchmark_key for this configured environment. +func getBenchmark() (*string, error) { + // Figure out what environment we're running in from the Mako configmap. + env, err := getEnvironment() + if err != nil { + return nil, err + } + // Read the Mako config file for this environment. + data, err := readConfigFromKoData(env) + if err != nil { + return nil, err + } + // Parse the Mako config file. + bi := &mpb.BenchmarkInfo{} + if err := proto.UnmarshalText(string(data), bi); err != nil { + return nil, err + } + + // Return the benchmark_key from this environment's config file. + return bi.BenchmarkKey, nil +} + +// readConfigFromKoData reads the named config file from kodata. +func readConfigFromKoData(environment string) ([]byte, error) { + koDataPath := os.Getenv(koDataPathEnvName) + if koDataPath == "" { + return nil, fmt.Errorf("%q does not exist or is empty", koDataPathEnvName) + } + fullFilename := filepath.Join(koDataPath, environment+".config") + return ioutil.ReadFile(fullFilename) +} diff --git a/vendor/knative.dev/pkg/test/mako/config.go b/vendor/knative.dev/pkg/test/mako/config.go new file mode 100644 index 00000000..c3e99740 --- /dev/null +++ b/vendor/knative.dev/pkg/test/mako/config.go @@ -0,0 +1,60 @@ +/* +Copyright 2019 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mako + +import ( + "strings" + + corev1 "k8s.io/api/core/v1" +) + +const ( + // ConfigName is the name of the config map for mako options. + ConfigName = "config-mako" +) + +// Config defines the mako configuration options. +type Config struct { + // Environment holds the name of the environement, + // where the test runs, e.g. `dev`. + Environment string + + // List of additional tags to apply to the run. + AdditionalTags []string +} + +// NewConfigFromMap creates a Config from the supplied map +func NewConfigFromMap(data map[string]string) (*Config, error) { + lc := &Config{ + Environment: "dev", + AdditionalTags: []string{}, + } + + if raw, ok := data["environment"]; ok { + lc.Environment = raw + } + if raw, ok := data["additionalTags"]; ok && raw != "" { + lc.AdditionalTags = strings.Split(raw, ",") + } + + return lc, nil +} + +// NewConfigFromConfigMap creates a Config from the supplied ConfigMap +func NewConfigFromConfigMap(configMap *corev1.ConfigMap) (*Config, error) { + return NewConfigFromMap(configMap.Data) +} diff --git a/vendor/knative.dev/pkg/test/mako/environment.go b/vendor/knative.dev/pkg/test/mako/environment.go new file mode 100644 index 00000000..2b5ecebb --- /dev/null +++ b/vendor/knative.dev/pkg/test/mako/environment.go @@ -0,0 +1,52 @@ +/* +Copyright 2019 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mako + +import ( + "log" + "path/filepath" + + "knative.dev/pkg/configmap" +) + +// TODO: perhaps cache the loaded CM. + +// MustGetTags returns the additional tags from the configmap, or dies. +func MustGetTags() []string { + makoCM, err := configmap.Load(filepath.Join("/etc", ConfigName)) + if err != nil { + log.Fatalf("unable to load configmap: %v", err) + } + cfg, err := NewConfigFromMap(makoCM) + if err != nil { + log.Fatalf("unable to parse configmap: %v", err) + } + return cfg.AdditionalTags +} + +// getEnvironment fetches the Mako config environment to which this cluster should publish. +func getEnvironment() (string, error) { + makoCM, err := configmap.Load(filepath.Join("/etc", ConfigName)) + if err != nil { + return "", err + } + cfg, err := NewConfigFromMap(makoCM) + if err != nil { + return "", err + } + return cfg.Environment, nil +} diff --git a/vendor/knative.dev/pkg/test/mako/sidecar.go b/vendor/knative.dev/pkg/test/mako/sidecar.go new file mode 100644 index 00000000..cbb0fdff --- /dev/null +++ b/vendor/knative.dev/pkg/test/mako/sidecar.go @@ -0,0 +1,96 @@ +/* +Copyright 2019 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mako + +import ( + "context" + "log" + "runtime" + "strings" + + "cloud.google.com/go/compute/metadata" + "knative.dev/pkg/injection/clients/kubeclient" + + "github.com/google/mako/helpers/go/quickstore" + qpb "github.com/google/mako/helpers/proto/quickstore/quickstore_go_proto" + "k8s.io/client-go/rest" + "knative.dev/pkg/changeset" + "knative.dev/pkg/controller" + "knative.dev/pkg/injection" +) + +const ( + // sidecarAddress is the address of the Mako sidecar to which we locally + // write results, and it authenticates and publishes them to Mako after + // assorted preprocessing. + sidecarAddress = "localhost:9813" +) + +// EscapeTag replaces characters that Mako doesn't accept with ones it does. +func EscapeTag(tag string) string { + return strings.ReplaceAll(tag, ".", "_") +} + +// Setup sets up the mako client for the provided benchmarkKey. +// It will add a few common tags and allow each benchmark to add custm tags as well. +// It returns the mako client handle to store metrics, a method to close the connection +// to mako server once done and error in case of failures. +func Setup(ctx context.Context, extraTags ...string) (context.Context, *quickstore.Quickstore, func(context.Context), error) { + tags := append(MustGetTags(), extraTags...) + // Get the commit of the benchmarks + commitID, err := changeset.Get() + if err != nil { + return nil, nil, nil, err + } + + // Setup a deployment informer, so that we can use the lister to track + // desired and available pod counts. + cfg, err := rest.InClusterConfig() + if err != nil { + return nil, nil, nil, err + } + ctx, informers := injection.Default.SetupInformers(ctx, cfg) + if err := controller.StartInformers(ctx.Done(), informers...); err != nil { + return nil, nil, nil, err + } + + // Get the Kubernetes version from the API server. + version, err := kubeclient.Get(ctx).Discovery().ServerVersion() + if err != nil { + return nil, nil, nil, err + } + + // Get GCP project ID as a tag. + if projectID, err := metadata.ProjectID(); err != nil { + log.Printf("GCP project ID is not available: %v", err) + } else { + tags = append(tags, "project-id="+EscapeTag(projectID)) + } + + qs, qclose, err := quickstore.NewAtAddress(ctx, &qpb.QuickstoreInput{ + BenchmarkKey: MustGetBenchmark(), + Tags: append(tags, + "commit="+commitID, + "kubernetes="+EscapeTag(version.String()), + EscapeTag(runtime.Version()), + ), + }, sidecarAddress) + if err != nil { + return nil, nil, nil, err + } + return ctx, qs, qclose, nil +} diff --git a/vendor/knative.dev/pkg/test/mako/testdata/config-mako.yaml b/vendor/knative.dev/pkg/test/mako/testdata/config-mako.yaml new file mode 100644 index 00000000..91b25069 --- /dev/null +++ b/vendor/knative.dev/pkg/test/mako/testdata/config-mako.yaml @@ -0,0 +1,47 @@ +# Copyright 2019 The Knative Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: ConfigMap +metadata: + name: config-mako + +data: + _example: | + ################################ + # # + # EXAMPLE CONFIGURATION # + # # + ################################ + + # This block is not actually functional configuration, + # but serves to illustrate the available configuration + # options and document them in a way that is accessible + # to users that `kubectl edit` this config map. + # + # These sample configuration options may be copied out of + # this example block and unindented to be in the data block + # to actually change the configuration. + + # The Mako environment in which we are running. + # Only our performance automation should run in "prod", but + # there should be a "dev" environment with a fairly broad + # write ACL. Users can also develop against custom configurations + # by adding `foo.config` under their benchmark's kodata directory. + environment: dev + + # Additional tags to tag the runs. These tags are added + # to the list that the binary itself publishes (Kubernetes version, etc). + # It is a comma separated list of tags. + additionalTags: "key=value,absolute" diff --git a/vendor/knative.dev/pkg/test/mako/time.go b/vendor/knative.dev/pkg/test/mako/time.go new file mode 100644 index 00000000..1a2f4cab --- /dev/null +++ b/vendor/knative.dev/pkg/test/mako/time.go @@ -0,0 +1,26 @@ +/* +Copyright 2019 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mako + +import ( + "time" +) + +// XTime converts a time.Time into a Mako x-axis compatible timestamp. +func XTime(t time.Time) float64 { + return float64(t.UnixNano()) / (1000.0 * 1000.0) +} diff --git a/vendor/knative.dev/pkg/testutils/clustermanager/gke.go b/vendor/knative.dev/pkg/testutils/clustermanager/gke.go index 033afe07..6c7a1df2 100644 --- a/vendor/knative.dev/pkg/testutils/clustermanager/gke.go +++ b/vendor/knative.dev/pkg/testutils/clustermanager/gke.go @@ -18,25 +18,77 @@ package clustermanager import ( "fmt" + "log" "strings" + "time" "knative.dev/pkg/testutils/clustermanager/boskos" "knative.dev/pkg/testutils/common" + + "golang.org/x/net/context" + "golang.org/x/oauth2/google" + "google.golang.org/api/container/v1" +) + +const ( + DefaultGKENumNodes = 1 + DefaultGKENodeType = "n1-standard-4" + DefaultGKERegion = "us-central1" + DefaultGKEZone = "" + regionEnv = "E2E_CLUSTER_REGION" + backupRegionEnv = "E2E_CLUSTER_BACKUP_REGIONS" +) + +var ( + DefaultGKEBackupRegions = []string{"us-west1", "us-east1"} + // This is an arbitrary number determined based on past experience + creationTimeout = 20 * time.Minute ) // GKEClient implements Client type GKEClient struct { } +// GKERequest contains all requests collected for cluster creation +type GKERequest struct { + NumNodes int64 + NodeType string + Region string + Zone string + BackupRegions []string +} + // GKECluster implements ClusterOperations type GKECluster struct { + Request *GKERequest // Project might be GKE specific, so put it here Project *string // NeedCleanup tells whether the cluster needs to be deleted afterwards // This probably should be part of task wrapper's logic NeedCleanup bool - // TODO: evaluate returning "google.golang.org/api/container/v1.Cluster" when implementing the creation logic - Cluster *string + Cluster *container.Cluster + operations GKESDKOperations +} + +// GKESDKOperations wraps GKE SDK related functions +type GKESDKOperations interface { + create(string, string, *container.CreateClusterRequest) (*container.Operation, error) + get(string, string, string) (*container.Cluster, error) +} + +// GKESDKClient Implement GKESDKOperations +type GKESDKClient struct { + *container.Service +} + +func (gsc *GKESDKClient) create(project, location string, rb *container.CreateClusterRequest) (*container.Operation, error) { + parent := fmt.Sprintf("projects/%s/locations/%s", project, location) + return gsc.Projects.Locations.Clusters.Create(parent, rb).Context(context.Background()).Do() +} + +func (gsc *GKESDKClient) get(project, location, cluster string) (*container.Cluster, error) { + clusterFullPath := fmt.Sprintf("projects/%s/locations/%s/clusters/%s", project, location, cluster) + return gsc.Projects.Locations.Clusters.Get(clusterFullPath).Context(context.Background()).Do() } // Setup sets up a GKECluster client. @@ -44,10 +96,31 @@ type GKECluster struct { // nodeType: default to n1-standard-4 if not provided // region: default to regional cluster if not provided, and use default backup regions // zone: default is none, must be provided together with region -func (gs *GKEClient) Setup(numNodes *int, nodeType *string, region *string, zone *string, project *string) (ClusterOperations, error) { - gc := &GKECluster{} - // check for local run - if nil != project { // if project is supplied, use it and create cluster +func (gs *GKEClient) Setup(numNodes *int64, nodeType *string, region *string, zone *string, project *string) (ClusterOperations, error) { + var err error + gc := &GKECluster{ + Request: &GKERequest{ + NumNodes: DefaultGKENumNodes, + NodeType: DefaultGKENodeType, + Region: DefaultGKERegion, + Zone: DefaultGKEZone, + BackupRegions: DefaultGKEBackupRegions}, + } + + ctx := context.Background() + + c, err := google.DefaultClient(ctx, container.CloudPlatformScope) + if nil != err { + return nil, fmt.Errorf("failed create google client: '%v'", err) + } + + containerService, err := container.New(c) + if nil != err { + return nil, fmt.Errorf("failed create container service: '%v'", err) + } + gc.operations = &GKESDKClient{containerService} + + if nil != project { // use provided project and create cluster gc.Project = project gc.NeedCleanup = true } else if err := gc.checkEnvironment(); nil != err { @@ -55,17 +128,38 @@ func (gs *GKEClient) Setup(numNodes *int, nodeType *string, region *string, zone } else if nil != gc.Cluster { // return if Cluster was already set by kubeconfig return gc, nil } - // check for Prow - if common.IsProw() { - boskosRes, err := boskos.AcquireGKEProject(nil) - if nil != err { - return nil, fmt.Errorf("failed acquire boskos project: '%v'", err) + if nil == gc.Cluster { + if common.IsProw() { + project, err := boskos.AcquireGKEProject(nil) + if nil != err { + return nil, fmt.Errorf("failed acquire boskos project: '%v'", err) + } + gc.Project = &project.Name + } + if nil != numNodes { + gc.Request.NumNodes = *numNodes + } + if nil != nodeType { + gc.Request.NodeType = *nodeType + } + if nil != region { + gc.Request.Region = *region + } + if "" != common.GetOSEnv(regionEnv) { + gc.Request.Region = common.GetOSEnv(regionEnv) + } + if "" != common.GetOSEnv(backupRegionEnv) { + gc.Request.BackupRegions = strings.Split(common.GetOSEnv(backupRegionEnv), " ") + } + if nil != zone { + gc.Request.Zone = *zone + gc.Request.BackupRegions = make([]string, 0) } - gc.Project = &boskosRes.Name } if nil == gc.Project || "" == *gc.Project { return nil, fmt.Errorf("gcp project must be set") } + log.Printf("use project '%s' for running test", *gc.Project) return gc, nil } @@ -74,14 +168,87 @@ func (gc *GKECluster) Provider() string { return "gke" } -// Acquire gets existing cluster or create a new one +// Acquire gets existing cluster or create a new one, the creation logic +// contains retries in BackupRegions. Default creating cluster +// in us-central1, and default BackupRegions are us-west1 and us-east1. If +// Region or Zone is provided then there is no retries func (gc *GKECluster) Acquire() error { + var err error // Check if using existing cluster if nil != gc.Cluster { return nil } - // TODO: Perform GKE specific cluster creation logics - return nil + // Perform GKE specific cluster creation logics + clusterName, err := getResourceName(ClusterResource) + if nil != err { + return fmt.Errorf("failed getting cluster name: '%v'", err) + } + + regions := []string{gc.Request.Region} + for _, br := range gc.Request.BackupRegions { + exist := false + for _, region := range regions { + if br == region { + exist = true + } + } + if !exist { + regions = append(regions, br) + } + } + var cluster *container.Cluster + for i, region := range regions { + rb := &container.CreateClusterRequest{ + Cluster: &container.Cluster{ + Name: clusterName, + InitialNodeCount: gc.Request.NumNodes, + NodeConfig: &container.NodeConfig{ + MachineType: gc.Request.NodeType, + }, + }, + ProjectId: *gc.Project, + } + log.Printf("Creating cluster in %s", getClusterLocation(region, gc.Request.Zone)) + _, err = gc.operations.create(*gc.Project, getClusterLocation(region, gc.Request.Zone), rb) + if nil == err { + // The process above doesn't seem to wait, wait for it + log.Printf("Waiting for cluster creation") + timeout := time.After(creationTimeout) + tick := time.Tick(50 * time.Millisecond) + for { + select { + // Got a timeout! fail with a timeout error + case <-timeout: + err = fmt.Errorf("timed out waiting for cluster creation") + break + case <-tick: + cluster, err = gc.operations.get(*gc.Project, getClusterLocation(region, gc.Request.Zone), clusterName) + } + if err != nil || cluster.Status == "RUNNING" { + break + } + if cluster.Status != "PROVISIONING" { + err = fmt.Errorf("cluster in bad state: '%s'", cluster.Status) + break + } + } + } + + if nil != err { + errMsg := fmt.Sprintf("error creating cluster: '%v'", err) + // TODO(chaodaiG): catch specific errors as we know what the error look like for stockout etc. + if len(regions) != i+1 { + errMsg = fmt.Sprintf("%s. Retry another region '%s' for cluster creation", errMsg, regions[i+1]) + } + log.Printf(errMsg) + } else { + log.Printf("cluster creation succeeded") + gc.Cluster = cluster + break + } + } + + return err } // Delete deletes a GKE cluster @@ -97,19 +264,28 @@ func (gc *GKECluster) Delete() error { // and sets up gc.Project and gc.Cluster properly, otherwise fail it. // if project can be derived from gcloud, sets it up as well func (gc *GKECluster) checkEnvironment() error { + var err error // if kubeconfig is configured, use it output, err := common.StandardExec("kubectl", "config", "current-context") if nil == err { - // output should be in the form of gke_PROJECT_REGION_CLUSTER - parts := strings.Split(string(output), "_") - if len(parts) != 4 { - return fmt.Errorf("kubectl current-context is malformed: '%s'", string(output)) + currentContext := strings.TrimSpace(string(output)) + if strings.HasPrefix(currentContext, "gke_") { + // output should be in the form of gke_PROJECT_REGION_CLUSTER + parts := strings.Split(currentContext, "_") + if len(parts) != 4 { // fall through with warning + log.Printf("WARNING: ignoring kubectl current-context since it's malformed: '%s'", currentContext) + } else { + log.Printf("kubeconfig isn't empty, uses this cluster for running tests: %s", currentContext) + gc.Project = &parts[1] + gc.Cluster, err = gc.operations.get(*gc.Project, parts[2], parts[3]) + if nil != err { + return fmt.Errorf("couldn't find cluster %s in %s in %s, does it exist? %v", parts[3], parts[1], parts[2], err) + } + return nil + } } - gc.Project = &parts[1] - gc.Cluster = &parts[3] - return nil } - if string(output) != "" { + if nil != err && len(output) > 0 { // this is unexpected error, should shout out directly return fmt.Errorf("failed running kubectl config current-context: '%s'", string(output)) } diff --git a/vendor/knative.dev/pkg/testutils/clustermanager/util.go b/vendor/knative.dev/pkg/testutils/clustermanager/util.go index 75302b1a..38ce790b 100644 --- a/vendor/knative.dev/pkg/testutils/clustermanager/util.go +++ b/vendor/knative.dev/pkg/testutils/clustermanager/util.go @@ -51,3 +51,10 @@ func getResourceName(rt ResourceType) (string, error) { } return resName, nil } + +func getClusterLocation(region, zone string) string { + if "" != zone { + region = fmt.Sprintf("%s-%s", region, zone) + } + return region +} diff --git a/vendor/knative.dev/test-infra/scripts/e2e-tests.sh b/vendor/knative.dev/test-infra/scripts/e2e-tests.sh index 6650b3a9..21133191 100755 --- a/vendor/knative.dev/test-infra/scripts/e2e-tests.sh +++ b/vendor/knative.dev/test-infra/scripts/e2e-tests.sh @@ -288,14 +288,10 @@ function create_test_cluster_with_retries() { [[ "$(get_test_return_code)" == "0" ]] && return 0 # Retry if cluster creation failed because of: # - stockout (https://github.com/knative/test-infra/issues/592) - # - latest GKE not available in this region/zone yet - # (https://github.com/knative/test-infra/issues/694) - # - cluster created but some nodes are unhealthy - # (https://github.com/knative/test-infra/issues/1291) + # - latest GKE not available in this region/zone yet (https://github.com/knative/test-infra/issues/694) [[ -z "$(grep -Fo 'does not have enough resources available to fulfill' ${cluster_creation_log})" \ && -z "$(grep -Fo 'ResponseError: code=400, message=No valid versions with the prefix' ${cluster_creation_log})" \ && -z "$(grep -Po 'ResponseError: code=400, message=Master version "[0-9a-z\-\.]+" is unsupported' ${cluster_creation_log})" ]] \ - && -z "$(grep -Po '[0-9]+ nodes out of [0-9]+ are unhealthy' ${cluster_creation_log})" ]] \ && return 1 done done diff --git a/vendor/knative.dev/test-infra/scripts/presubmit-tests.sh b/vendor/knative.dev/test-infra/scripts/presubmit-tests.sh index e0cf9d19..18d785ac 100755 --- a/vendor/knative.dev/test-infra/scripts/presubmit-tests.sh +++ b/vendor/knative.dev/test-infra/scripts/presubmit-tests.sh @@ -172,14 +172,19 @@ function default_build_test_runner() { # Get all build tags in go code (ignore /vendor) local tags="$(grep -r '// +build' . \ | grep -v '^./vendor/' | cut -f3 -d' ' | sort | uniq | tr '\n' ' ')" - if [[ -n "${tags}" ]]; then - errors="" - if ! capture_output "${report}" go test -run=^$ -tags="${tags}" ./... ; then + local tagged_pkgs="$(grep -r '// +build' . \ + | grep -v '^./vendor/' | grep ":// +build " | cut -f1 -d: | xargs dirname | sort | uniq | tr '\n' ' ')" + for pkg in ${tagged_pkgs}; do + # `go test -c` lets us compile the tests but do not run them. + if ! capture_output "${report}" go test -c -tags="${tags}" ${pkg} ; then failed=1 # Consider an error message everything that's not a successful test result. - errors_go2="$(grep -v '^\(ok\|\?\)\s\+\(github\.com\|knative\.dev\)/' "${report}")" + errors_go2+="$(grep -v '^\(ok\|\?\)\s\+\(github\.com\|knative\.dev\)/' "${report}")" fi - fi + # Remove unused generated binary, if any. + rm -f e2e.test + done + local errors_go="$(echo -e "${errors_go1}\n${errors_go2}" | uniq)" create_junit_xml _build_tests Build_Go "${errors_go}" if [[ -f ./hack/verify-codegen.sh ]]; then