diff --git a/Gopkg.lock b/Gopkg.lock index 4dda1a1c..32aaaccc 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -927,7 +927,7 @@ [[projects]] branch = "master" - digest = "1:2a1af5e20affdf0508f4296328c812f1448bff1c5d7a8d74a2a8a56b5913ff72" + digest = "1:2f0bfe47d6d494dfc206af0e4f4e7000db3fef052ed46026c4c6a2ba4c7cac32" name = "knative.dev/pkg" packages = [ "apis", @@ -946,18 +946,18 @@ "metrics/metricskey", ] pruneopts = "T" - revision = "849fcc967b598b47945067cfe0536377bac6bd19" + revision = "0b19b4ad91391699170edab6b36f7545c5ca6ead" [[projects]] branch = "master" - digest = "1:8009d9ea7464198279c5c1b1a760365c1f731aa88239632c08929df5b7f323f6" + digest = "1:0041ff0e6be31ac2cd85ffd4cf0b6ccb576b408d4f3c1c7e48c79cbf32a1319c" name = "knative.dev/test-infra" packages = [ "scripts", "tools/dep-collector", ] pruneopts = "UT" - revision = "12f3c6a839ace07e8171d00e3057c11ed33c465c" + revision = "8923e6806b082ebc75b5464ad23d9826f8eaa97b" [[projects]] digest = "1:8730e0150dfb2b7e173890c8b9868e7a273082ef8e39f4940e3506a481cf895c" diff --git a/vendor/knative.dev/pkg/.gitattributes b/vendor/knative.dev/pkg/.gitattributes index 95f18a11..b1e71af9 100644 --- a/vendor/knative.dev/pkg/.gitattributes +++ b/vendor/knative.dev/pkg/.gitattributes @@ -4,7 +4,6 @@ **/zz_generated.*.go linguist-generated=true /client/** linguist-generated=true -/test/** coverage-excluded=true /metrics/gcp_metadata.go coverage-excluded=true *.sh text eol=lf diff --git a/vendor/knative.dev/pkg/Gopkg.lock b/vendor/knative.dev/pkg/Gopkg.lock index bf72094f..8635e8ef 100644 --- a/vendor/knative.dev/pkg/Gopkg.lock +++ b/vendor/knative.dev/pkg/Gopkg.lock @@ -2,13 +2,18 @@ [[projects]] - digest = "1:5f43842d8fe08b43ada82d57e48a844800b4163d1150f7f451e81cb347fccb72" + digest = "1:a0c6f9487d58c98483e8785cc45e181c3cb7a21549f9f32c2e80fedb7141db18" name = "cloud.google.com/go" packages = [ "compute/metadata", "container/apiv1", + "iam", + "internal", + "internal/optional", + "internal/trace", "internal/version", "monitoring/apiv3", + "storage", "trace/apiv2", ] pruneopts = "NUT" @@ -709,7 +714,7 @@ [[projects]] branch = "master" - digest = "1:3b865f5fdc62cc6f52865a3bf29146a384fd5b2f250d8d86c0c2a41a4ffa10c4" + digest = "1:48871545d029f0561819b6f761e11ed3111e79f29f7f1ba36e2e428e05ceaa9d" name = "google.golang.org/api" packages = [ "container/v1beta1", @@ -720,6 +725,7 @@ "internal", "iterator", "option", + "storage/v1", "support/bundler", "transport", "transport/grpc", @@ -752,7 +758,7 @@ [[projects]] branch = "master" - digest = "1:0ee5f291bbeb4c9664aad14ad9a64e52cdeed406c97549beb6028adb9d7b8afc" + digest = "1:445a47089d0c1013621851ca7fa2992a499871e7b239fc3e904c404ab72f6a04" name = "google.golang.org/genproto" packages = [ "googleapis/api", @@ -763,8 +769,11 @@ "googleapis/api/monitoredres", "googleapis/container/v1", "googleapis/devtools/cloudtrace/v2", + "googleapis/iam/v1", "googleapis/monitoring/v3", + "googleapis/rpc/code", "googleapis/rpc/status", + "googleapis/type/expr", "protobuf/field_mask", ] pruneopts = "NUT" @@ -832,12 +841,12 @@ version = "v0.9.1" [[projects]] - digest = "1:18108594151654e9e696b27b181b953f9a90b16bf14d253dd1b397b025a1487f" + digest = "1:accc3bfe4e404aa53ac3621470e7cf9fce1efe48f0fabcfe6d12a72579d9d91f" name = "gopkg.in/yaml.v2" packages = ["."] pruneopts = "NUT" - revision = "51d6538a90f86fe93ac480b35f37b2be17fef232" - version = "v2.2.2" + revision = "f221b8435cfb71e54062f6c6e99e9ade30b124d5" + version = "v2.2.4" [[projects]] digest = "1:2b55f57d7b6d9f0f23478f2432b03273073f1bf93aed18620472e7117eb4451e" @@ -1273,6 +1282,7 @@ analyzer-version = 1 input-imports = [ "cloud.google.com/go/compute/metadata", + "cloud.google.com/go/storage", "contrib.go.opencensus.io/exporter/prometheus", "contrib.go.opencensus.io/exporter/stackdriver", "contrib.go.opencensus.io/exporter/stackdriver/monitoredresource", @@ -1280,7 +1290,6 @@ "github.com/davecgh/go-spew/spew", "github.com/evanphx/json-patch", "github.com/ghodss/yaml", - "github.com/golang/glog", "github.com/golang/protobuf/jsonpb", "github.com/golang/protobuf/proto", "github.com/google/go-cmp/cmp", @@ -1320,6 +1329,8 @@ "golang.org/x/oauth2/google", "golang.org/x/sync/errgroup", "google.golang.org/api/container/v1beta1", + "google.golang.org/api/iterator", + "google.golang.org/api/option", "google.golang.org/grpc", "gopkg.in/yaml.v2", "k8s.io/api/admission/v1beta1", diff --git a/vendor/knative.dev/pkg/Gopkg.toml b/vendor/knative.dev/pkg/Gopkg.toml index e7064f6a..7fc55e76 100644 --- a/vendor/knative.dev/pkg/Gopkg.toml +++ b/vendor/knative.dev/pkg/Gopkg.toml @@ -35,6 +35,10 @@ required = [ name = "k8s.io/code-generator" version = "kubernetes-1.15.3" +[[override]] + name = "gopkg.in/yaml.v2" + version = "v2.2.4" + [[override]] name = "github.com/json-iterator/go" # This is the commit at which k8s depends on this in 1.11 diff --git a/vendor/knative.dev/pkg/OWNERS_ALIASES b/vendor/knative.dev/pkg/OWNERS_ALIASES index 1eb19f22..52cfdb2b 100644 --- a/vendor/knative.dev/pkg/OWNERS_ALIASES +++ b/vendor/knative.dev/pkg/OWNERS_ALIASES @@ -46,6 +46,11 @@ aliases: - mdemirhan - yanweiguo + network-approvers: + - markusthoemmes + - tcnghia + - vagababov + productivity-approvers: - adrcunha - chaodaiG @@ -57,6 +62,10 @@ aliases: - steuhs - yt3liu + source-approvers: + - n3wscott + - vaikas-google + webhook-approvers: - mattmoor - grantr diff --git a/vendor/knative.dev/pkg/apis/url.go b/vendor/knative.dev/pkg/apis/url.go index f5d8d034..b0599075 100644 --- a/vendor/knative.dev/pkg/apis/url.go +++ b/vendor/knative.dev/pkg/apis/url.go @@ -41,6 +41,30 @@ func ParseURL(u string) (*URL, error) { return (*URL)(pu), nil } +// HTTP creates an http:// URL pointing to a known domain. +func HTTP(domain string) *URL { + return &URL{ + Scheme: "http", + Host: domain, + } +} + +// HTTPS creates an https:// URL pointing to a known domain. +func HTTPS(domain string) *URL { + return &URL{ + Scheme: "https", + Host: domain, + } +} + +// IsEmpty returns true if the URL is `nil` or represents an empty URL. +func (u *URL) IsEmpty() bool { + if u == nil { + return true + } + return *u == URL{} +} + // MarshalJSON implements a custom json marshal method used when this type is // marshaled using json.Marshal. // json.Marshaler impl @@ -79,6 +103,9 @@ func (u *URL) String() string { // URL returns the URL as a url.URL. func (u *URL) URL() *url.URL { + if u == nil { + return &url.URL{} + } url := url.URL(*u) return &url } diff --git a/vendor/knative.dev/pkg/apis/v1alpha1/destination.go b/vendor/knative.dev/pkg/apis/v1alpha1/destination.go index 0f9ee409..dae152de 100644 --- a/vendor/knative.dev/pkg/apis/v1alpha1/destination.go +++ b/vendor/knative.dev/pkg/apis/v1alpha1/destination.go @@ -18,132 +18,46 @@ package v1alpha1 import ( "context" - "net/url" - "path" - "strings" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/equality" - "knative.dev/pkg/apis" ) // Destination represents a target of an invocation over HTTP. type Destination struct { - // ObjectReference points to an Addressable. - *corev1.ObjectReference `json:",inline"` + // Ref points to an Addressable. + // +optional + Ref *corev1.ObjectReference `json:"ref,omitempty"` - // URI is for direct URI Designations. + // URI can be an absolute URL(non-empty scheme and non-empty host) pointing to the target or a relative URI. Relative URIs will be resolved using the base URI retrieved from Ref. + // +optional URI *apis.URL `json:"uri,omitempty"` - - // Path is used with the resulting URL from Addressable ObjectReference or URI. Must start - // with `/`. An empty path should be represented as the nil value, not `` or `/`. Will be - // appended to the path of the resulting URL from the Addressable, or URI. - Path *string `json:"path,omitempty"` -} - -// NewDestination constructs a Destination from an object reference as a convenience. -func NewDestination(obj *corev1.ObjectReference, paths ...string) (*Destination, error) { - dest := &Destination{ - ObjectReference: obj, - } - err := dest.AppendPath(paths...) - if err != nil { - return nil, err - } - return dest, nil -} - -// NewDestinationURI constructs a Destination from a URI. -func NewDestinationURI(uri *apis.URL, paths ...string) (*Destination, error) { - dest := &Destination{ - URI: uri, - } - err := dest.AppendPath(paths...) - if err != nil { - return nil, err - } - return dest, nil -} - -// AppendPath iteratively appends paths to the Destination. -// The path will always begin with "/" unless it is empty. -// An empty path ("" or "/") will always resolve to nil. -func (current *Destination) AppendPath(paths ...string) error { - // Start with empty string or existing path - var fullpath string - if current.Path != nil { - fullpath = *current.Path - } - - // Intelligently join all the paths provided - fullpath = path.Join("/", fullpath, path.Join(paths...)) - - // Parse the URL to trim garbage - urlpath, err := apis.ParseURL(fullpath) - if err != nil { - return err - } - - // apis.ParseURL returns nil if our path was empty, then our path - // should reflect that it is not set. - if urlpath == nil { - current.Path = nil - return nil - } - - // A path of "/" adds no information, just toss it - // Note that urlpath.Path == "" is always false here (joined with "/" above). - if urlpath.Path == "/" { - current.Path = nil - return nil - } - - // Only use the plain path from the URL - current.Path = &urlpath.Path - return nil } func (current *Destination) Validate(ctx context.Context) *apis.FieldError { if current != nil { - errs := validateDestination(*current).ViaField(apis.CurrentField) - if current.Path != nil { - errs = errs.Also(validateDestinationPath(*current.Path).ViaField("path")) - } - return errs + return ValidateDestination(*current).ViaField(apis.CurrentField) } else { return nil } } -func validateDestination(dest Destination) *apis.FieldError { - if dest.URI != nil { - if dest.ObjectReference != nil { - return apis.ErrMultipleOneOf("uri", "[apiVersion, kind, name]") +func ValidateDestination(dest Destination) *apis.FieldError { + if dest.Ref == nil && dest.URI == nil { + return apis.ErrGeneric("expected at least one, got neither", "ref", "uri") + } + if dest.Ref != nil && dest.URI != nil && dest.URI.URL().IsAbs() { + return apis.ErrGeneric("Absolute URI is not allowed when Ref is present", "ref", "uri") + } + // IsAbs() check whether the URL has a non-empty scheme. Besides the non-empty scheme, we also require dest.URI has a non-empty host + if dest.Ref == nil && dest.URI != nil && (!dest.URI.URL().IsAbs() || dest.URI.Host == "") { + return apis.ErrInvalidValue("Relative URI is not allowed when Ref is absent", "uri") } - if dest.URI.Host == "" || dest.URI.Scheme == "" { - return apis.ErrInvalidValue(dest.URI.String(), "uri") - } - } else if dest.ObjectReference == nil { - return apis.ErrMissingOneOf("uri", "[apiVersion, kind, name]") - } else { - return validateDestinationRef(*dest.ObjectReference) + if dest.Ref != nil && dest.URI == nil{ + return validateDestinationRef(*dest.Ref).ViaField("ref") } return nil } -func validateDestinationPath(path string) *apis.FieldError { - if strings.HasPrefix(path, "/") { - if pu, err := url.Parse(path); err != nil { - return apis.ErrInvalidValue(path, apis.CurrentField) - } else if !equality.Semantic.DeepEqual(pu, &url.URL{Path: pu.Path}) { - return apis.ErrInvalidValue(path, apis.CurrentField) - } - } else { - return apis.ErrInvalidValue(path, apis.CurrentField) - } - return nil -} func validateDestinationRef(ref corev1.ObjectReference) *apis.FieldError { // Check the object. diff --git a/vendor/knative.dev/pkg/apis/v1alpha1/zz_generated.deepcopy.go b/vendor/knative.dev/pkg/apis/v1alpha1/zz_generated.deepcopy.go index a54dcace..5640bf0a 100644 --- a/vendor/knative.dev/pkg/apis/v1alpha1/zz_generated.deepcopy.go +++ b/vendor/knative.dev/pkg/apis/v1alpha1/zz_generated.deepcopy.go @@ -28,8 +28,8 @@ import ( // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Destination) DeepCopyInto(out *Destination) { *out = *in - if in.ObjectReference != nil { - in, out := &in.ObjectReference, &out.ObjectReference + if in.Ref != nil { + in, out := &in.Ref, &out.Ref *out = new(v1.ObjectReference) **out = **in } @@ -38,11 +38,6 @@ func (in *Destination) DeepCopyInto(out *Destination) { *out = new(apis.URL) (*in).DeepCopyInto(*out) } - if in.Path != nil { - in, out := &in.Path, &out.Path - *out = new(string) - **out = **in - } return } diff --git a/vendor/knative.dev/pkg/controller/controller.go b/vendor/knative.dev/pkg/controller/controller.go index 0b8be1c6..38fafda8 100644 --- a/vendor/knative.dev/pkg/controller/controller.go +++ b/vendor/knative.dev/pkg/controller/controller.go @@ -445,6 +445,27 @@ func StartInformers(stopCh <-chan struct{}, informers ...Informer) error { return nil } +// RunInformers kicks off all of the passed informers and then waits for all of +// them to synchronize. Returned function will wait for all informers to finish. +func RunInformers(stopCh <-chan struct{}, informers ...Informer) (func(), error) { + var wg sync.WaitGroup + wg.Add(len(informers)) + for _, informer := range informers { + informer := informer + go func() { + defer wg.Done() + informer.Run(stopCh) + }() + } + + for i, informer := range informers { + if ok := cache.WaitForCacheSync(stopCh, informer.HasSynced); !ok { + return wg.Wait, fmt.Errorf("failed to wait for cache at index %d to sync", i) + } + } + return wg.Wait, nil +} + // StartAll kicks off all of the passed controllers with DefaultThreadsPerController. func StartAll(stopCh <-chan struct{}, controllers ...*Impl) { wg := sync.WaitGroup{} diff --git a/vendor/knative.dev/pkg/logging/config.go b/vendor/knative.dev/pkg/logging/config.go index a08e0fae..2317c927 100644 --- a/vendor/knative.dev/pkg/logging/config.go +++ b/vendor/knative.dev/pkg/logging/config.go @@ -33,7 +33,12 @@ import ( const ConfigMapNameEnv = "CONFIG_LOGGING_NAME" -var zapLoggerConfig = "zap-logger-config" +const ( + loggerConfigKey = "zap-logger-config" + fallbackLoggerName = "fallback-logger" +) + +var emptyLoggerConfigError = errors.New("empty logger configuration") // NewLogger creates a logger with the supplied configuration. // In addition to the logger, it returns AtomicLevel that can @@ -48,7 +53,7 @@ func NewLogger(configJSON string, levelOverride string, opts ...zap.Option) (*za } loggingCfg := zap.NewProductionConfig() - if len(levelOverride) > 0 { + if levelOverride != "" { if level, err := levelFromString(levelOverride); err == nil { loggingCfg.Level = zap.NewAtomicLevelAt(*level) } @@ -58,7 +63,7 @@ func NewLogger(configJSON string, levelOverride string, opts ...zap.Option) (*za if err2 != nil { panic(err2) } - return enrichLoggerWithCommitID(logger.Named("fallback-logger").Sugar()), loggingCfg.Level + return enrichLoggerWithCommitID(logger.Named(fallbackLoggerName).Sugar()), loggingCfg.Level } func enrichLoggerWithCommitID(logger *zap.SugaredLogger) *zap.SugaredLogger { @@ -74,21 +79,22 @@ func enrichLoggerWithCommitID(logger *zap.SugaredLogger) *zap.SugaredLogger { // NewLoggerFromConfig creates a logger using the provided Config func NewLoggerFromConfig(config *Config, name string, opts ...zap.Option) (*zap.SugaredLogger, zap.AtomicLevel) { - logger, level := NewLogger(config.LoggingConfig, config.LoggingLevel[name].String(), opts...) + var componentLvl string + if lvl, defined := config.LoggingLevel[name]; defined { + componentLvl = lvl.String() + } + + logger, level := NewLogger(config.LoggingConfig, componentLvl, opts...) return logger.Named(name), level } func newLoggerFromConfig(configJSON string, levelOverride string, opts []zap.Option) (*zap.Logger, zap.AtomicLevel, error) { - if len(configJSON) == 0 { - return nil, zap.AtomicLevel{}, errors.New("empty logging configuration") - } - - var loggingCfg zap.Config - if err := json.Unmarshal([]byte(configJSON), &loggingCfg); err != nil { + loggingCfg, err := zapConfigFromJSON(configJSON) + if err != nil { return nil, zap.AtomicLevel{}, err } - if len(levelOverride) > 0 { + if levelOverride != "" { if level, err := levelFromString(levelOverride); err == nil { loggingCfg.Level = zap.NewAtomicLevelAt(*level) } @@ -104,6 +110,18 @@ func newLoggerFromConfig(configJSON string, levelOverride string, opts []zap.Opt return logger, loggingCfg.Level, nil } +func zapConfigFromJSON(configJSON string) (*zap.Config, error) { + if configJSON == "" { + return nil, emptyLoggerConfigError + } + + loggingCfg := &zap.Config{} + if err := json.Unmarshal([]byte(configJSON), loggingCfg); err != nil { + return nil, err + } + return loggingCfg, nil +} + // Config contains the configuration defined in the logging ConfigMap. // +k8s:deepcopy-gen=true type Config struct { @@ -136,7 +154,7 @@ const defaultZLC = `{ // expecting the given list of components. func NewConfigFromMap(data map[string]string) (*Config, error) { lc := &Config{} - if zlc, ok := data["zap-logger-config"]; ok { + if zlc, ok := data[loggerConfigKey]; ok { lc.LoggingConfig = zlc } else { lc.LoggingConfig = defaultZLC @@ -157,7 +175,7 @@ func NewConfigFromMap(data map[string]string) (*Config, error) { return lc, nil } -// NewConfigFromConfigMap creates a LoggingConfig from the supplied ConfigMap, +// NewConfigFromConfigMap creates a Config from the supplied ConfigMap, // expecting the given list of components. func NewConfigFromConfigMap(configMap *corev1.ConfigMap) (*Config, error) { return NewConfigFromMap(configMap.Data) @@ -175,14 +193,30 @@ func levelFromString(level string) (*zapcore.Level, error) { // when a config map is updated func UpdateLevelFromConfigMap(logger *zap.SugaredLogger, atomicLevel zap.AtomicLevel, levelKey string) func(configMap *corev1.ConfigMap) { + return func(configMap *corev1.ConfigMap) { - loggingConfig, err := NewConfigFromConfigMap(configMap) + config, err := NewConfigFromConfigMap(configMap) if err != nil { logger.Errorw("Failed to parse the logging configmap. Previous config map will be used.", zap.Error(err)) return } - level := loggingConfig.LoggingLevel[levelKey] + level, defined := config.LoggingLevel[levelKey] + if !defined { + // reset to global level + loggingCfg, err := zapConfigFromJSON(config.LoggingConfig) + switch { + case err == emptyLoggerConfigError: + level = zap.NewAtomicLevel().Level() + case err != nil: + logger.With(zap.Error(err)).Errorf("Failed to parse logger configuration. "+ + "Previous log level retained for %v", levelKey) + return + default: + level = loggingCfg.Level.Level() + } + } + if atomicLevel.Level() != level { logger.Infof("Updating logging level for %v from %v to %v.", levelKey, atomicLevel.Level(), level) atomicLevel.SetLevel(level) @@ -228,7 +262,7 @@ func LoggingConfigToJson(cfg *Config) (string, error) { } jsonCfg, err := json.Marshal(map[string]string{ - zapLoggerConfig: cfg.LoggingConfig, + loggerConfigKey: cfg.LoggingConfig, }) if err != nil { return "", err diff --git a/vendor/knative.dev/pkg/metrics/config.go b/vendor/knative.dev/pkg/metrics/config.go index ebfa5b35..3e3d922f 100644 --- a/vendor/knative.dev/pkg/metrics/config.go +++ b/vendor/knative.dev/pkg/metrics/config.go @@ -27,7 +27,6 @@ import ( "time" "go.uber.org/zap" - corev1 "k8s.io/api/core/v1" ) const ( @@ -60,28 +59,6 @@ const ( minPrometheusPort = 1024 ) -// ExporterOptions contains options for configuring the exporter. -type ExporterOptions struct { - // Domain is the metrics domain. e.g. "knative.dev". Must be present. - Domain string - - // Component is the name of the component that emits the metrics. e.g. - // "activator", "queue_proxy". Should only contains alphabets and underscore. - // Must be present. - Component string - - // PrometheusPort is the port to expose metrics if metrics backend is Prometheus. - // It should be between maxPrometheusPort and maxPrometheusPort. 0 value means - // using the default 9090 value. If is ignored if metrics backend is not - // Prometheus. - PrometheusPort int - - // ConfigMap is the data from config map config-observability. Must be present. - // See https://github.com/knative/serving/blob/master/config/config-observability.yaml - // for details. - ConfigMap map[string]string -} - type metricsConfig struct { // The metrics domain. e.g. "serving.knative.dev" or "build.knative.dev". domain string @@ -125,7 +102,7 @@ type metricsConfig struct { stackdriverCustomMetricTypePrefix string } -func getMetricsConfig(ops ExporterOptions, logger *zap.SugaredLogger) (*metricsConfig, error) { +func createMetricsConfig(ops ExporterOptions, logger *zap.SugaredLogger) (*metricsConfig, error) { var mc metricsConfig if ops.Domain == "" { @@ -215,62 +192,6 @@ func getMetricsConfig(ops ExporterOptions, logger *zap.SugaredLogger) (*metricsC return &mc, nil } -// UpdateExporterFromConfigMap returns a helper func that can be used to update the exporter -// when a config map is updated. -func UpdateExporterFromConfigMap(component string, logger *zap.SugaredLogger) func(configMap *corev1.ConfigMap) { - domain := Domain() - return func(configMap *corev1.ConfigMap) { - UpdateExporter(ExporterOptions{ - Domain: domain, - Component: component, - ConfigMap: configMap.Data, - }, logger) - } -} - -// UpdateExporter updates the exporter based on the given ExporterOptions. -func UpdateExporter(ops ExporterOptions, logger *zap.SugaredLogger) error { - newConfig, err := getMetricsConfig(ops, logger) - if err != nil { - if ce := getCurMetricsExporter(); ce == nil { - // Fail the process if there doesn't exist an exporter. - logger.Errorw("Failed to get a valid metrics config", zap.Error(err)) - } else { - logger.Errorw("Failed to get a valid metrics config; Skip updating the metrics exporter", zap.Error(err)) - } - return err - } - - if isNewExporterRequired(newConfig) { - logger.Info("Flushing the existing exporter before setting up the new exporter.") - FlushExporter() - e, err := newMetricsExporter(newConfig, logger) - if err != nil { - logger.Errorf("Failed to update a new metrics exporter based on metric config %v. error: %v", newConfig, err) - return err - } - existingConfig := getCurMetricsConfig() - setCurMetricsExporter(e) - logger.Infof("Successfully updated the metrics exporter; old config: %v; new config %v", existingConfig, newConfig) - } - - setCurMetricsConfig(newConfig) - return nil -} - -// isNewExporterRequired compares the non-nil newConfig against curMetricsConfig. When backend changes, -// or stackdriver project ID changes for stackdriver backend, we need to update the metrics exporter. -func isNewExporterRequired(newConfig *metricsConfig) bool { - cc := getCurMetricsConfig() - if cc == nil || newConfig.backendDestination != cc.backendDestination { - return true - } else if newConfig.backendDestination == Stackdriver && newConfig.stackdriverProjectID != cc.stackdriverProjectID { - return true - } - - return false -} - // ConfigMapName gets the name of the metrics ConfigMap func ConfigMapName() string { cm := os.Getenv(ConfigMapNameEnv) diff --git a/vendor/knative.dev/pkg/metrics/exporter.go b/vendor/knative.dev/pkg/metrics/exporter.go index 0c97c9b9..22cebecf 100644 --- a/vendor/knative.dev/pkg/metrics/exporter.go +++ b/vendor/knative.dev/pkg/metrics/exporter.go @@ -19,12 +19,13 @@ import ( "go.opencensus.io/stats/view" "go.uber.org/zap" + corev1 "k8s.io/api/core/v1" ) var ( curMetricsExporter view.Exporter curMetricsConfig *metricsConfig - metricsMux sync.Mutex + metricsMux sync.RWMutex ) type flushable interface { @@ -32,11 +33,102 @@ type flushable interface { Flush() } +// ExporterOptions contains options for configuring the exporter. +type ExporterOptions struct { + // Domain is the metrics domain. e.g. "knative.dev". Must be present. + // + // Stackdriver uses the following format to construct full metric name: + // // + // Prometheus uses the following format to construct full metric name: + // _ + // Domain is actually not used if metrics backend is Prometheus. + Domain string + + // Component is the name of the component that emits the metrics. e.g. + // "activator", "queue_proxy". Should only contains alphabets and underscore. + // Must be present. + Component string + + // PrometheusPort is the port to expose metrics if metrics backend is Prometheus. + // It should be between maxPrometheusPort and maxPrometheusPort. 0 value means + // using the default 9090 value. If is ignored if metrics backend is not + // Prometheus. + PrometheusPort int + + // ConfigMap is the data from config map config-observability. Must be present. + // See https://github.com/knative/serving/blob/master/config/config-observability.yaml + // for details. + ConfigMap map[string]string +} + +// UpdateExporterFromConfigMap returns a helper func that can be used to update the exporter +// when a config map is updated. +func UpdateExporterFromConfigMap(component string, logger *zap.SugaredLogger) func(configMap *corev1.ConfigMap) { + domain := Domain() + return func(configMap *corev1.ConfigMap) { + UpdateExporter(ExporterOptions{ + Domain: domain, + Component: component, + ConfigMap: configMap.Data, + }, logger) + } +} + +// UpdateExporter updates the exporter based on the given ExporterOptions. +// This is a thread-safe function. The entire series of operations is locked +// to prevent a race condition between reading the current configuration +// and updating the current exporter. +func UpdateExporter(ops ExporterOptions, logger *zap.SugaredLogger) error { + metricsMux.Lock() + defer metricsMux.Unlock() + + newConfig, err := createMetricsConfig(ops, logger) + if err != nil { + if curMetricsExporter == nil { + // Fail the process if there doesn't exist an exporter. + logger.Errorw("Failed to get a valid metrics config", zap.Error(err)) + } else { + logger.Errorw("Failed to get a valid metrics config; Skip updating the metrics exporter", zap.Error(err)) + } + return err + } + + if isNewExporterRequired(newConfig) { + logger.Info("Flushing the existing exporter before setting up the new exporter.") + flushExporterUnlocked(curMetricsExporter) + e, err := newMetricsExporter(newConfig, logger) + if err != nil { + logger.Errorf("Failed to update a new metrics exporter based on metric config %v. error: %v", newConfig, err) + return err + } + existingConfig := curMetricsConfig + setCurMetricsExporterUnlocked(e) + logger.Infof("Successfully updated the metrics exporter; old config: %v; new config %v", existingConfig, newConfig) + } + + setCurMetricsConfigUnlocked(newConfig) + return nil +} + +// isNewExporterRequired compares the non-nil newConfig against curMetricsConfig. When backend changes, +// or stackdriver project ID changes for stackdriver backend, we need to update the metrics exporter. +// This function is not implicitly thread-safe. +func isNewExporterRequired(newConfig *metricsConfig) bool { + cc := curMetricsConfig + if cc == nil || newConfig.backendDestination != cc.backendDestination { + return true + } + + return newConfig.backendDestination == Stackdriver && newConfig.stackdriverProjectID != cc.stackdriverProjectID +} + // newMetricsExporter gets a metrics exporter based on the config. +// This function is not implicitly thread-safe. func newMetricsExporter(config *metricsConfig, logger *zap.SugaredLogger) (view.Exporter, error) { + ce := curMetricsExporter // If there is a Prometheus Exporter server running, stop it. resetCurPromSrv() - ce := getCurMetricsExporter() + if ce != nil { // UnregisterExporter is idempotent and it can be called multiple times for the same exporter // without side effects. @@ -59,27 +151,35 @@ func newMetricsExporter(config *metricsConfig, logger *zap.SugaredLogger) (view. } func getCurMetricsExporter() view.Exporter { - metricsMux.Lock() - defer metricsMux.Unlock() + metricsMux.RLock() + defer metricsMux.RUnlock() return curMetricsExporter } func setCurMetricsExporter(e view.Exporter) { metricsMux.Lock() defer metricsMux.Unlock() + setCurMetricsExporterUnlocked(e) +} + +func setCurMetricsExporterUnlocked(e view.Exporter) { view.RegisterExporter(e) curMetricsExporter = e } func getCurMetricsConfig() *metricsConfig { - metricsMux.Lock() - defer metricsMux.Unlock() + metricsMux.RLock() + defer metricsMux.RUnlock() return curMetricsConfig } func setCurMetricsConfig(c *metricsConfig) { metricsMux.Lock() defer metricsMux.Unlock() + setCurMetricsConfigUnlocked(c) +} + +func setCurMetricsConfigUnlocked(c *metricsConfig) { if c != nil { view.SetReportingPeriod(c.reportingPeriod) } else { @@ -94,6 +194,10 @@ func setCurMetricsConfig(c *metricsConfig) { // Return value indicates whether the exporter is flushable or not. func FlushExporter() bool { e := getCurMetricsExporter() + return flushExporterUnlocked(e) +} + +func flushExporterUnlocked(e view.Exporter) bool { if e == nil { return false } diff --git a/vendor/knative.dev/pkg/reconciler/testing/table.go b/vendor/knative.dev/pkg/reconciler/testing/table.go index 151acf06..3a4d8daf 100644 --- a/vendor/knative.dev/pkg/reconciler/testing/table.go +++ b/vendor/knative.dev/pkg/reconciler/testing/table.go @@ -89,6 +89,9 @@ type TableRow struct { // For cluster-scoped resources like ClusterIngress, it does not have to be // in the same namespace with its child resources. SkipNamespaceValidation bool + + // PostConditions allows custom assertions to be made after reconciliation + PostConditions []func(*testing.T, *TableRow) } func objKey(o runtime.Object) string { @@ -331,6 +334,10 @@ func (r *TableRow) Test(t *testing.T, factory Factory) { if diff := cmp.Diff(r.WantServiceReadyStats, gotStats); diff != "" { t.Errorf("Unexpected service ready stats (-want, +got): %s", diff) } + + for _, verify := range r.PostConditions { + verify(t, r) + } } func filterUpdatesWithSubresource( diff --git a/vendor/knative.dev/pkg/resolver/addressable_resolver.go b/vendor/knative.dev/pkg/resolver/addressable_resolver.go index dec16dac..b2181234 100644 --- a/vendor/knative.dev/pkg/resolver/addressable_resolver.go +++ b/vendor/knative.dev/pkg/resolver/addressable_resolver.go @@ -20,7 +20,6 @@ import ( "context" "errors" "fmt" - "path" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" @@ -65,20 +64,29 @@ func NewURIResolver(ctx context.Context, callback func(types.NamespacedName)) *U // URIFromDestination resolves a Destination into a URI string. func (r *URIResolver) URIFromDestination(dest apisv1alpha1.Destination, parent interface{}) (string, error) { - // Prefer resolved object reference + path, then try URI + path, honoring the Destination documentation - if dest.ObjectReference != nil { - url, err := r.URIFromObjectReference(dest.ObjectReference, parent) + if dest.Ref != nil { + url, err := r.URIFromObjectReference(dest.Ref, parent) if err != nil { return "", err } - return extendPath(url.DeepCopy(), dest.Path).String(), nil + if dest.URI != nil { + if dest.URI.URL().IsAbs() { + return "", fmt.Errorf("absolute URI is not allowed when Ref exists") + } + return url.URL().ResolveReference(dest.URI.URL()).String(), nil + } + return url.URL().String(), nil } if dest.URI != nil { - return extendPath(dest.URI.DeepCopy(), dest.Path).String(), nil + // IsAbs check whether the URL has a non-empty scheme. Besides the non non-empty scheme, we also require dest.URI has a non-empty host + if !dest.URI.URL().IsAbs() || dest.URI.Host == "" { + return "", fmt.Errorf("URI is not absolute(both scheme and host should be non-empty): %v", dest.URI.String()) + } + return dest.URI.String(), nil } - return "", fmt.Errorf("destination missing ObjectReference and URI, expected exactly one") + return "", fmt.Errorf("destination missing Ref and URI, expected at least one") } // URIFromObjectReference resolves an ObjectReference to a URI string. @@ -131,16 +139,6 @@ func (r *URIResolver) URIFromObjectReference(ref *corev1.ObjectReference, parent return url, nil } -// extendPath is a convenience wrapper to add a destination's path. -func extendPath(url *apis.URL, extrapath *string) *apis.URL { - if extrapath == nil { - return url - } - - url.Path = path.Join(url.Path, *extrapath) - return url -} - // ServiceHostName resolves the hostname for a Kubernetes Service. func ServiceHostName(serviceName, namespace string) string { return fmt.Sprintf("%s.%s.svc.%s", serviceName, namespace, network.GetClusterDomainName()) diff --git a/vendor/knative.dev/pkg/source/doc.go b/vendor/knative.dev/pkg/source/doc.go new file mode 100644 index 00000000..fc827c38 --- /dev/null +++ b/vendor/knative.dev/pkg/source/doc.go @@ -0,0 +1,18 @@ +/* +Copyright 2019 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package source holds utilities for Source developers. +package source diff --git a/vendor/knative.dev/pkg/source/source_labels.go b/vendor/knative.dev/pkg/source/source_labels.go new file mode 100644 index 00000000..f669bad6 --- /dev/null +++ b/vendor/knative.dev/pkg/source/source_labels.go @@ -0,0 +1,29 @@ +/* + * Copyright 2019 The Knative Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package source + +const ( + sourceControllerName = "sources.knative.dev/controller" + sourceName = "sources.knative.dev/name" +) + +func Labels(name, controllerAgentName string) map[string]string { + return map[string]string{ + sourceControllerName: controllerAgentName, + sourceName: name, + } +} diff --git a/vendor/knative.dev/pkg/test/gcs/gcs.go b/vendor/knative.dev/pkg/test/gcs/gcs.go new file mode 100644 index 00000000..d243015b --- /dev/null +++ b/vendor/knative.dev/pkg/test/gcs/gcs.go @@ -0,0 +1,224 @@ +/* +Copyright 2019 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// gcs.go defines functions to use GCS + +package gcs + +import ( + "context" + "fmt" + "io" + "io/ioutil" + "log" + "net/url" + "os" + "path" + "strings" + + "cloud.google.com/go/storage" + "google.golang.org/api/iterator" + "google.golang.org/api/option" +) + +var client *storage.Client + +// Authenticate explicitly sets up authentication for the rest of run +func Authenticate(ctx context.Context, serviceAccount string) error { + var err error + client, err = storage.NewClient(ctx, option.WithCredentialsFile(serviceAccount)) + return err +} + +// Exists checks if path exist under gcs bucket, +// this path can either be a directory or a file. +func Exists(ctx context.Context, bucketName, storagePath string) bool { + // Check if this is a file + handle := createStorageObject(bucketName, storagePath) + if _, err := handle.Attrs(ctx); err == nil { + return true + } + // Check if this is a directory, + // gcs directory paths are virtual paths, they automatically got deleted if there is no child file + _, err := getObjectsIter(ctx, bucketName, strings.TrimRight(storagePath, " /")+"/", "").Next() + return err == nil +} + +// ListChildrenFiles recursively lists all children files. +func ListChildrenFiles(ctx context.Context, bucketName, storagePath string) []string { + return list(ctx, bucketName, strings.TrimRight(storagePath, " /")+"/", "") +} + +// ListDirectChildren lists direct children paths (including files and directories). +func ListDirectChildren(ctx context.Context, bucketName, storagePath string) []string { + // If there are 2 directories named "foo" and "foobar", + // then given storagePath "foo" will get files both under "foo" and "foobar". + // Add trailling slash to storagePath, so that only gets children under given directory. + return list(ctx, bucketName, strings.TrimRight(storagePath, " /")+"/", "/") +} + +// Copy file from within gcs +func Copy(ctx context.Context, srcBucketName, srcPath, dstBucketName, dstPath string) error { + src := createStorageObject(srcBucketName, srcPath) + dst := createStorageObject(dstBucketName, dstPath) + + _, err := dst.CopierFrom(src).Run(ctx) + return err +} + +// Download file from gcs +func Download(ctx context.Context, bucketName, srcPath, dstPath string) error { + handle := createStorageObject(bucketName, srcPath) + if _, err := handle.Attrs(ctx); err != nil { + return err + } + + dst, err := os.OpenFile(dstPath, os.O_RDWR|os.O_CREATE, 0755) + if err != nil { + return err + } + src, err := handle.NewReader(ctx) + if err != nil { + return err + } + defer src.Close() + _, err = io.Copy(dst, src) + return err +} + +// Upload file to gcs +func Upload(ctx context.Context, bucketName, dstPath, srcPath string) error { + src, err := os.Open(srcPath) + if err != nil { + return err + } + dst := createStorageObject(bucketName, dstPath).NewWriter(ctx) + defer dst.Close() + _, err = io.Copy(dst, src) + return err +} + +// Read reads the specified file +func Read(ctx context.Context, bucketName, filePath string) ([]byte, error) { + var contents []byte + f, err := NewReader(ctx, bucketName, filePath) + if err != nil { + return contents, err + } + defer f.Close() + return ioutil.ReadAll(f) +} + +// ReadURL reads from a gsUrl and return a log structure +func ReadURL(ctx context.Context, gcsURL string) ([]byte, error) { + bucket, obj, err := linkToBucketAndObject(gcsURL) + if err != nil { + return nil, err + } + + return Read(ctx, bucket, obj) +} + +// NewReader creates a new Reader of a gcs file. +// Important: caller must call Close on the returned Reader when done reading +func NewReader(ctx context.Context, bucketName, filePath string) (*storage.Reader, error) { + o := createStorageObject(bucketName, filePath) + if _, err := o.Attrs(ctx); err != nil { + return nil, err + } + return o.NewReader(ctx) +} + +// BuildLogPath returns the build log path from the test result gcsURL +func BuildLogPath(gcsURL string) (string, error) { + u, err := url.Parse(gcsURL) + if err != nil { + return gcsURL, err + } + u.Path = path.Join(u.Path, "build-log.txt") + return u.String(), nil +} + +// GetConsoleURL returns the gcs link renderable directly from a browser +func GetConsoleURL(gcsURL string) (string, error) { + u, err := url.Parse(gcsURL) + if err != nil { + return gcsURL, err + } + u.Path = path.Join("storage/browser", u.Host, u.Path) + u.Scheme = "https" + u.Host = "console.cloud.google.com" + return u.String(), nil +} + +// create storage object handle, this step doesn't access internet +func createStorageObject(bucketName, filePath string) *storage.ObjectHandle { + return client.Bucket(bucketName).Object(filePath) +} + +// Query items under given gcs storagePath, use exclusionFilter to eliminate some files. +func getObjectsAttrs(ctx context.Context, bucketName, storagePath, exclusionFilter string) []*storage.ObjectAttrs { + var allAttrs []*storage.ObjectAttrs + it := getObjectsIter(ctx, bucketName, storagePath, exclusionFilter) + + for { + attrs, err := it.Next() + if err == iterator.Done { + break + } + if err != nil { + log.Fatalf("Error iterating: %v", err) + } + allAttrs = append(allAttrs, attrs) + } + return allAttrs +} + +// list child under storagePath, use exclusionFilter for skipping some files. +// This function gets all child files recursively under given storagePath, +// then filter out filenames containing giving exclusionFilter. +// If exclusionFilter is empty string, returns all files but not directories, +// if exclusionFilter is "/", returns all direct children, including both files and directories. +// see https://godoc.org/cloud.google.com/go/storage#Query +func list(ctx context.Context, bucketName, storagePath, exclusionFilter string) []string { + var filePaths []string + objsAttrs := getObjectsAttrs(ctx, bucketName, storagePath, exclusionFilter) + for _, attrs := range objsAttrs { + filePaths = append(filePaths, path.Join(attrs.Prefix, attrs.Name)) + } + return filePaths +} + +// get objects iterator under given storagePath and bucketName, use exclusionFilter to eliminate some files. +func getObjectsIter(ctx context.Context, bucketName, storagePath, exclusionFilter string) *storage.ObjectIterator { + return client.Bucket(bucketName).Objects(ctx, &storage.Query{ + Prefix: storagePath, + Delimiter: exclusionFilter, + }) +} + +// get the bucket and object from the gsURL +func linkToBucketAndObject(gsURL string) (string, string, error) { + var bucket, obj string + gsURL = strings.Replace(gsURL, "gs://", "", 1) + + sIdx := strings.IndexByte(gsURL, '/') + if sIdx == -1 || sIdx+1 >= len(gsURL) { + return bucket, obj, fmt.Errorf("the gsUrl (%s) cannot be converted to bucket/object", gsURL) + } + + return gsURL[:sIdx], gsURL[sIdx+1:], nil +} diff --git a/vendor/knative.dev/pkg/test/gke/addon.go b/vendor/knative.dev/pkg/test/gke/addon.go new file mode 100644 index 00000000..f8f98d02 --- /dev/null +++ b/vendor/knative.dev/pkg/test/gke/addon.go @@ -0,0 +1,55 @@ +/* +Copyright 2019 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gke + +import ( + "fmt" + "strings" + + container "google.golang.org/api/container/v1beta1" +) + +const ( + // Define all supported addons here + istio = "istio" + hpa = "horizontalpodautoscaling" + hlb = "httploadbalancing" + cloudRun = "cloudrun" +) + +// GetAddonsConfig gets AddonsConfig from a slice of addon names, contains the logic of +// converting string argument to typed AddonsConfig, for example `IstioConfig`. +// Currently supports Istio, HorizontalPodAutoscaling, HttpLoadBalancing and CloudRun. +func GetAddonsConfig(addons []string) *container.AddonsConfig { + ac := &container.AddonsConfig{} + for _, name := range addons { + switch strings.ToLower(name) { + case istio: + ac.IstioConfig = &container.IstioConfig{Disabled: false} + case hpa: + ac.HorizontalPodAutoscaling = &container.HorizontalPodAutoscaling{Disabled: false} + case hlb: + ac.HttpLoadBalancing = &container.HttpLoadBalancing{Disabled: false} + case cloudRun: + ac.CloudRunConfig = &container.CloudRunConfig{Disabled: false} + default: + panic(fmt.Sprintf("addon type %q not supported. Has to be one of: %q", name, []string{istio, hpa, hlb, cloudRun})) + } + } + + return ac +} diff --git a/vendor/knative.dev/pkg/test/gke/client.go b/vendor/knative.dev/pkg/test/gke/client.go new file mode 100644 index 00000000..9d76bd78 --- /dev/null +++ b/vendor/knative.dev/pkg/test/gke/client.go @@ -0,0 +1,132 @@ +/* +Copyright 2019 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gke + +import ( + "fmt" + + container "google.golang.org/api/container/v1beta1" + + "golang.org/x/net/context" + "golang.org/x/oauth2/google" +) + +// SDKOperations wraps GKE SDK related functions +type SDKOperations interface { + CreateCluster(project, region, zone string, req *container.CreateClusterRequest) error + CreateClusterAsync(project, region, zone string, req *container.CreateClusterRequest) (*container.Operation, error) + DeleteCluster(project, region, zone, clusterName string) error + DeleteClusterAsync(project, region, zone, clusterName string) (*container.Operation, error) + GetCluster(project, region, zone, clusterName string) (*container.Cluster, error) + GetOperation(project, region, zone, opName string) (*container.Operation, error) + ListClustersInProject(project string) ([]*container.Cluster, error) +} + +// sdkClient Implement SDKOperations +type sdkClient struct { + *container.Service +} + +// NewSDKClient returns an SDKClient that implements SDKOperations +func NewSDKClient() (SDKOperations, error) { + ctx := context.Background() + c, err := google.DefaultClient(ctx, container.CloudPlatformScope) + if err != nil { + return nil, fmt.Errorf("failed to create Google client: '%v'", err) + } + + containerService, err := container.New(c) + if err != nil { + return nil, fmt.Errorf("failed to create container service: '%v'", err) + } + return &sdkClient{containerService}, nil +} + +// CreateCluster creates a new GKE cluster, and wait until it finishes or timeout or there is an error. +func (gsc *sdkClient) CreateCluster( + project, region, zone string, + rb *container.CreateClusterRequest, +) error { + op, err := gsc.CreateClusterAsync(project, region, zone, rb) + if err == nil { + err = Wait(gsc, project, region, zone, op.Name, creationTimeout) + } + return err +} + +// CreateClusterAsync creates a new GKE cluster asynchronously. +func (gsc *sdkClient) CreateClusterAsync( + project, region, zone string, + rb *container.CreateClusterRequest, +) (*container.Operation, error) { + location := GetClusterLocation(region, zone) + if zone != "" { + return gsc.Projects.Zones.Clusters.Create(project, location, rb).Context(context.Background()).Do() + } + parent := fmt.Sprintf("projects/%s/locations/%s", project, location) + return gsc.Projects.Locations.Clusters.Create(parent, rb).Context(context.Background()).Do() +} + +// DeleteCluster deletes the GKE cluster, and wait until it finishes or timeout or there is an error. +func (gsc *sdkClient) DeleteCluster(project, region, zone, clusterName string) error { + op, err := gsc.DeleteClusterAsync(project, region, zone, clusterName) + if err == nil { + err = Wait(gsc, project, region, zone, op.Name, deletionTimeout) + } + return err +} + +// DeleteClusterAsync deletes the GKE cluster asynchronously. +func (gsc *sdkClient) DeleteClusterAsync(project, region, zone, clusterName string) (*container.Operation, error) { + location := GetClusterLocation(region, zone) + if zone != "" { + return gsc.Projects.Zones.Clusters.Delete(project, location, clusterName).Context(context.Background()).Do() + } + clusterFullPath := fmt.Sprintf("projects/%s/locations/%s/clusters/%s", project, location, clusterName) + return gsc.Projects.Locations.Clusters.Delete(clusterFullPath).Context(context.Background()).Do() +} + +// GetCluster gets the GKE cluster with the given cluster name. +func (gsc *sdkClient) GetCluster(project, region, zone, clusterName string) (*container.Cluster, error) { + location := GetClusterLocation(region, zone) + if zone != "" { + return gsc.Projects.Zones.Clusters.Get(project, location, clusterName).Context(context.Background()).Do() + } + clusterFullPath := fmt.Sprintf("projects/%s/locations/%s/clusters/%s", project, location, clusterName) + return gsc.Projects.Locations.Clusters.Get(clusterFullPath).Context(context.Background()).Do() +} + +// ListClustersInProject lists all the GKE clusters created in the given project. +func (gsc *sdkClient) ListClustersInProject(project string) ([]*container.Cluster, error) { + var clusters []*container.Cluster + projectFullPath := fmt.Sprintf("projects/%s/locations/-", project) + resp, err := gsc.Projects.Locations.Clusters.List(projectFullPath).Do() + if err != nil { + return clusters, fmt.Errorf("failed to list clusters under project %s: %v", project, err) + } + return resp.Clusters, nil +} + +// GetOperation gets the operation ref with the given operation name. +func (gsc *sdkClient) GetOperation(project, region, zone, opName string) (*container.Operation, error) { + location := GetClusterLocation(region, zone) + if zone != "" { + return gsc.Service.Projects.Zones.Operations.Get(project, location, opName).Do() + } + opsFullPath := fmt.Sprintf("projects/%s/locations/%s/operations/%s", project, location, opName) + return gsc.Service.Projects.Locations.Operations.Get(opsFullPath).Do() +} diff --git a/vendor/knative.dev/pkg/test/gke/fake/client.go b/vendor/knative.dev/pkg/test/gke/fake/client.go new file mode 100644 index 00000000..c3af1f3c --- /dev/null +++ b/vendor/knative.dev/pkg/test/gke/fake/client.go @@ -0,0 +1,205 @@ +/* +Copyright 2019 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package fake + +import ( + "errors" + "fmt" + "strconv" + "strings" + "sync" + "time" + + container "google.golang.org/api/container/v1beta1" + "knative.dev/pkg/test/gke" +) + +// Timeout for fake client. +// Need to be changed dynamically in the tests, so make them public. +var ( + CreationTimeout = 1000 * time.Millisecond + DeletionTimeout = 10 * time.Minute +) + +// GKESDKClient is a fake client for unit tests. +type GKESDKClient struct { + // map of parent: clusters slice + clusters map[string][]*container.Cluster + // map of operationID: operation + ops map[string]*container.Operation + + // An incremental number for new ops + opNumber int + // A lookup table for determining ops statuses + OpStatus map[string]string + + mutex sync.Mutex +} + +// NewGKESDKClient returns a new fake gkeSDKClient that can be used in unit tests. +func NewGKESDKClient() *GKESDKClient { + return &GKESDKClient{ + clusters: make(map[string][]*container.Cluster), + ops: make(map[string]*container.Operation), + OpStatus: make(map[string]string), + } +} + +// automatically registers new ops, and mark it "DONE" by default. Update +// fgsc.opStatus by fgsc.opStatus[string(fgsc.opNumber+1)]="PENDING" to make the +// next operation pending +func (fgsc *GKESDKClient) newOp() *container.Operation { + opName := strconv.Itoa(fgsc.opNumber) + op := &container.Operation{ + Name: opName, + Status: "DONE", + } + if status, ok := fgsc.OpStatus[opName]; ok { + op.Status = status + } + fgsc.opNumber++ + fgsc.ops[opName] = op + return op +} + +// CreateCluster creates a new cluster, and wait until it finishes or timeout or there is an error. +func (fgsc *GKESDKClient) CreateCluster( + project, region, zone string, + rb *container.CreateClusterRequest, +) error { + op, err := fgsc.CreateClusterAsync(project, region, zone, rb) + if err == nil { + err = gke.Wait(fgsc, project, region, zone, op.Name, CreationTimeout) + } + return err +} + +// CreateClusterAsync creates a new cluster asynchronously. +func (fgsc *GKESDKClient) CreateClusterAsync( + project, region, zone string, + rb *container.CreateClusterRequest, +) (*container.Operation, error) { + fgsc.mutex.Lock() + defer fgsc.mutex.Unlock() + location := gke.GetClusterLocation(region, zone) + parent := fmt.Sprintf("projects/%s/locations/%s", project, location) + name := rb.Cluster.Name + if cls, ok := fgsc.clusters[parent]; ok { + for _, cl := range cls { + if cl.Name == name { + return nil, errors.New("cluster already exist") + } + } + } else { + fgsc.clusters[parent] = make([]*container.Cluster, 0) + } + cluster := &container.Cluster{ + Name: name, + Location: location, + Status: "RUNNING", + AddonsConfig: rb.Cluster.AddonsConfig, + NodePools: rb.Cluster.NodePools, + } + if rb.Cluster.NodePools != nil { + cluster.NodePools = rb.Cluster.NodePools + } + if rb.Cluster.MasterAuth != nil { + cluster.MasterAuth = &container.MasterAuth{ + Username: rb.Cluster.MasterAuth.Username, + } + } + + fgsc.clusters[parent] = append(fgsc.clusters[parent], cluster) + return fgsc.newOp(), nil +} + +// DeleteCluster deletes the cluster, and wait until it finishes or timeout or there is an error. +func (fgsc *GKESDKClient) DeleteCluster( + project, region, zone, clusterName string, +) error { + op, err := fgsc.DeleteClusterAsync(project, region, zone, clusterName) + if err == nil { + err = gke.Wait(fgsc, project, region, zone, op.Name, DeletionTimeout) + } + return err +} + +// DeleteClusterAsync deletes the cluster asynchronously. +func (fgsc *GKESDKClient) DeleteClusterAsync( + project, region, zone, clusterName string, +) (*container.Operation, error) { + fgsc.mutex.Lock() + defer fgsc.mutex.Unlock() + location := gke.GetClusterLocation(region, zone) + parent := fmt.Sprintf("projects/%s/locations/%s", project, location) + found := -1 + if clusters, ok := fgsc.clusters[parent]; ok { + for i, cluster := range clusters { + if cluster.Name == clusterName { + found = i + } + } + } + if found == -1 { + return nil, fmt.Errorf("cluster %q not found for deletion", clusterName) + } + // Delete this cluster + fgsc.clusters[parent] = append(fgsc.clusters[parent][:found], fgsc.clusters[parent][found+1:]...) + return fgsc.newOp(), nil +} + +// GetCluster gets the cluster with the given settings. +func (fgsc *GKESDKClient) GetCluster(project, region, zone, cluster string) (*container.Cluster, error) { + fgsc.mutex.Lock() + defer fgsc.mutex.Unlock() + location := gke.GetClusterLocation(region, zone) + parent := fmt.Sprintf("projects/%s/locations/%s", project, location) + if cls, ok := fgsc.clusters[parent]; ok { + for _, cl := range cls { + if cl.Name == cluster { + return cl, nil + } + } + } + return nil, fmt.Errorf("cluster not found") +} + +// ListClustersInProject lists all the GKE clusters created in the given project. +func (fgsc *GKESDKClient) ListClustersInProject(project string) ([]*container.Cluster, error) { + fgsc.mutex.Lock() + defer fgsc.mutex.Unlock() + allClusters := make([]*container.Cluster, 0) + projectPath := fmt.Sprintf("projects/%s", project) + for location, cls := range fgsc.clusters { + // If the clusters are under this project + if strings.HasPrefix(location, projectPath) { + allClusters = append(allClusters, cls...) + } + } + return allClusters, nil +} + +// GetOperation gets the operation with the given settings. +func (fgsc *GKESDKClient) GetOperation(project, region, zone, opName string) (*container.Operation, error) { + fgsc.mutex.Lock() + op, ok := fgsc.ops[opName] + fgsc.mutex.Unlock() + if ok { + return op, nil + } + return nil, errors.New(opName + " operation not found") +} diff --git a/vendor/knative.dev/pkg/test/gke/location.go b/vendor/knative.dev/pkg/test/gke/location.go new file mode 100644 index 00000000..36f99f35 --- /dev/null +++ b/vendor/knative.dev/pkg/test/gke/location.go @@ -0,0 +1,44 @@ +/* +Copyright 2019 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gke + +import ( + "fmt" + "strings" +) + +// GetClusterLocation returns the location used in GKE operations, given the region and zone. +func GetClusterLocation(region, zone string) string { + if zone != "" { + region = fmt.Sprintf("%s-%s", region, zone) + } + return region +} + +// RegionZoneFromLoc returns the region and the zone, given the location. +func RegionZoneFromLoc(location string) (string, string) { + parts := strings.Split(location, "-") + // zonal location is the form of us-central1-a, and this pattern is + // consistent in all available GCP locations so far, so we are looking for + // location with more than 2 "-" + if len(parts) > 2 { + zone := parts[len(parts)-1] + region := strings.TrimRight(location, "-"+zone) + return region, zone + } + return location, "" +} diff --git a/vendor/knative.dev/pkg/test/gke/request.go b/vendor/knative.dev/pkg/test/gke/request.go new file mode 100644 index 00000000..9a64afe3 --- /dev/null +++ b/vendor/knative.dev/pkg/test/gke/request.go @@ -0,0 +1,122 @@ +/* +Copyright 2019 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gke + +import ( + "errors" + + container "google.golang.org/api/container/v1beta1" +) + +const defaultGKEVersion = "latest" + +// Request contains all settings collected for cluster creation +type Request struct { + // Project: name of the gcloud project for the cluster + Project string + + // GKEVersion: GKE version of the cluster, default to be latest if not provided + GKEVersion string + + // ClusterName: name of the cluster + ClusterName string + + // MinNodes: the minimum number of nodes of the cluster + MinNodes int64 + + // MaxNodes: the maximum number of nodes of the cluster + MaxNodes int64 + + // NodeType: node type of the cluster, e.g. n1-standard-4, n1-standard-8 + NodeType string + + // Region: region of the cluster, e.g. us-west1, us-central1 + Region string + + // Zone: default is none, must be provided together with region + Zone string + + // Addons: cluster addons to be added to cluster, such as istio + Addons []string +} + +// DeepCopy will make a deepcopy of the request struct. +func (r *Request) DeepCopy() *Request { + return &Request{ + Project: r.Project, + GKEVersion: r.GKEVersion, + ClusterName: r.ClusterName, + MinNodes: r.MinNodes, + MaxNodes: r.MaxNodes, + NodeType: r.NodeType, + Region: r.Region, + Zone: r.Zone, + Addons: r.Addons, + } +} + +// NewCreateClusterRequest returns a new CreateClusterRequest that can be used in gcloud SDK. +func NewCreateClusterRequest(request *Request) (*container.CreateClusterRequest, error) { + if request.ClusterName == "" { + return nil, errors.New("cluster name cannot be empty") + } + if request.MinNodes <= 0 { + return nil, errors.New("min nodes must be larger than 1") + } + if request.MinNodes > request.MaxNodes { + return nil, errors.New("min nodes cannot be larger than max nodes") + } + if request.NodeType == "" { + return nil, errors.New("node type cannot be empty") + } + + if request.GKEVersion == "" { + request.GKEVersion = defaultGKEVersion + } + + return &container.CreateClusterRequest{ + Cluster: &container.Cluster{ + NodePools: []*container.NodePool{ + { + Name: "default-pool", + InitialNodeCount: request.MinNodes, + Autoscaling: &container.NodePoolAutoscaling{ + Enabled: true, + MinNodeCount: request.MinNodes, + MaxNodeCount: request.MaxNodes, + }, + Config: &container.NodeConfig{ + MachineType: request.NodeType, + }, + }, + }, + Name: request.ClusterName, + // The default cluster version is not latest, has to explicitly + // set it as "latest" + InitialClusterVersion: request.GKEVersion, + // Installing addons after cluster creation takes at least 5 + // minutes, so install addons as part of cluster creation, which + // doesn't seem to add much time on top of cluster creation + AddonsConfig: GetAddonsConfig(request.Addons), + // Equivalent to --enable-basic-auth, so that user:pass can be + // later on retrieved for setting up cluster roles. Use the + // default username from gcloud command, the password will be + // automatically generated by GKE SDK + MasterAuth: &container.MasterAuth{Username: "admin"}, + }, + }, nil +} diff --git a/vendor/knative.dev/pkg/test/gke/wait.go b/vendor/knative.dev/pkg/test/gke/wait.go new file mode 100644 index 00000000..e53d2ef6 --- /dev/null +++ b/vendor/knative.dev/pkg/test/gke/wait.go @@ -0,0 +1,76 @@ +/* +Copyright 2019 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gke + +import ( + "errors" + "fmt" + "time" + + container "google.golang.org/api/container/v1beta1" +) + +// These are arbitrary numbers determined based on past experience +var ( + creationTimeout = 20 * time.Minute + deletionTimeout = 10 * time.Minute +) + +const ( + pendingStatus = "PENDING" + runningStatus = "RUNNING" + doneStatus = "DONE" +) + +// Wait depends on unique opName(operation ID created by cloud), and waits until +// it's done +func Wait(gsc SDKOperations, project, region, zone, opName string, wait time.Duration) error { + var op *container.Operation + var err error + + timeout := time.After(wait) + tick := time.Tick(500 * time.Millisecond) + for { + select { + // Got a timeout! fail with a timeout error + case <-timeout: + return errors.New("timed out waiting") + case <-tick: + // Retry 3 times in case of weird network error, or rate limiting + for r, w := 0, 50*time.Microsecond; r < 3; r, w = r+1, w*2 { + op, err = gsc.GetOperation(project, region, zone, opName) + if err == nil { + if op.Status == doneStatus { + return nil + } else if op.Status == pendingStatus || op.Status == runningStatus { + // Valid operation, no need to retry + break + } else { + // Have seen intermittent error state and fixed itself, + // let it retry to avoid too much flakiness + err = fmt.Errorf("unexpected operation status: %q", op.Status) + } + } + time.Sleep(w) + } + // If err still persist after retries, exit + if err != nil { + return err + } + } + } +} diff --git a/vendor/knative.dev/pkg/test/helpers/data.go b/vendor/knative.dev/pkg/test/helpers/data.go deleted file mode 100644 index e982b2c3..00000000 --- a/vendor/knative.dev/pkg/test/helpers/data.go +++ /dev/null @@ -1,77 +0,0 @@ -/* -Copyright 2019 The Knative Authors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package helpers - -import ( - "math/rand" - "strings" - "time" - "unicode" -) - -const ( - letterBytes = "abcdefghijklmnopqrstuvwxyz" - randSuffixLen = 8 - sep = '-' -) - -func init() { - rand.Seed(time.Now().UTC().UnixNano()) -} - -// AppendRandomString will generate a random string that begins with prefix. -// This is useful if you want to make sure that your tests can run at the same -// time against the same environment without conflicting. -// This method will use "-" as the separator between the prefix and -// the random suffix. -// This method will seed rand with the current time when the package is initialized. -func AppendRandomString(prefix string) string { - suffix := make([]byte, randSuffixLen) - - for i := range suffix { - suffix[i] = letterBytes[rand.Intn(len(letterBytes))] - } - - return strings.Join([]string{prefix, string(suffix)}, string(sep)) -} - -// MakeK8sNamePrefix converts each chunk of non-alphanumeric character into a single dash -// and also convert camelcase tokens into dash-delimited lowercase tokens. -func MakeK8sNamePrefix(s string) string { - var sb strings.Builder - newToken := false - for _, c := range s { - if !(unicode.IsLetter(c) || unicode.IsNumber(c)) { - newToken = true - continue - } - if sb.Len() > 0 && (newToken || unicode.IsUpper(c)) { - sb.WriteRune(sep) - } - sb.WriteRune(unicode.ToLower(c)) - newToken = false - } - return sb.String() -} - -// GetBaseFuncName returns the baseFuncName parsed from the fullFuncName. -// eg. test/e2e.TestMain will return TestMain. -func GetBaseFuncName(fullFuncName string) string { - baseFuncName := fullFuncName[strings.LastIndex(fullFuncName, "/")+1:] - baseFuncName = baseFuncName[strings.LastIndex(baseFuncName, ".")+1:] - return baseFuncName -} diff --git a/vendor/knative.dev/pkg/test/helpers/run.go b/vendor/knative.dev/pkg/test/helpers/dryrun.go similarity index 100% rename from vendor/knative.dev/pkg/test/helpers/run.go rename to vendor/knative.dev/pkg/test/helpers/dryrun.go diff --git a/vendor/knative.dev/pkg/test/helpers/name.go b/vendor/knative.dev/pkg/test/helpers/name.go index 50ad03a8..5014814f 100644 --- a/vendor/knative.dev/pkg/test/helpers/name.go +++ b/vendor/knative.dev/pkg/test/helpers/name.go @@ -17,14 +17,30 @@ limitations under the License. package helpers import ( + "log" + "math/rand" "strings" "testing" + "time" + "unicode" ) const ( + letterBytes = "abcdefghijklmnopqrstuvwxyz" + randSuffixLen = 8 + sep = '-' testNamePrefix = "Test" ) +func init() { + // Properly seed the random number generator so AppendRandomString() is actually random. + // Otherwise, rerunning tests will generate the same names for the test resources, causing conflicts with + // already existing resources. + seed := time.Now().UTC().UnixNano() + log.Printf("Using '%d' to seed the random number generator", seed) + rand.Seed(seed) +} + // ObjectPrefixForTest returns the name prefix for this test's random names. func ObjectPrefixForTest(t *testing.T) string { return MakeK8sNamePrefix(strings.TrimPrefix(t.Name(), testNamePrefix)) @@ -34,3 +50,49 @@ func ObjectPrefixForTest(t *testing.T) string { func ObjectNameForTest(t *testing.T) string { return AppendRandomString(ObjectPrefixForTest(t)) } + +// AppendRandomString will generate a random string that begins with prefix. +// This is useful if you want to make sure that your tests can run at the same +// time against the same environment without conflicting. +// This method will use "-" as the separator between the prefix and +// the random suffix. +func AppendRandomString(prefix string) string { + suffix := make([]byte, randSuffixLen) + + for i := range suffix { + suffix[i] = letterBytes[rand.Intn(len(letterBytes))] + } + + return strings.Join([]string{prefix, string(suffix)}, string(sep)) +} + +// MakeK8sNamePrefix converts each chunk of non-alphanumeric character into a single dash +// and also convert camelcase tokens into dash-delimited lowercase tokens. +func MakeK8sNamePrefix(s string) string { + var sb strings.Builder + newToken := false + for _, c := range s { + if !(unicode.IsLetter(c) || unicode.IsNumber(c)) { + newToken = true + continue + } + if sb.Len() > 0 && (newToken || unicode.IsUpper(c)) { + sb.WriteRune(sep) + } + sb.WriteRune(unicode.ToLower(c)) + newToken = false + } + return sb.String() +} + +// GetBaseFuncName returns the baseFuncName parsed from the fullFuncName. +// eg. test/e2e.TestMain will return TestMain. +func GetBaseFuncName(fullFuncName string) string { + name := fullFuncName + // Possibly there is no parent package, so only remove it from the name if '/' exists + if strings.ContainsRune(name, '/') { + name = name[strings.LastIndex(name, "/")+1:] + } + name = name[strings.LastIndex(name, ".")+1:] + return name +} diff --git a/vendor/knative.dev/pkg/test/logging/logging.go b/vendor/knative.dev/pkg/test/logging/logging.go index 4d9c5a96..74ef1d24 100644 --- a/vendor/knative.dev/pkg/test/logging/logging.go +++ b/vendor/knative.dev/pkg/test/logging/logging.go @@ -21,13 +21,11 @@ package logging import ( "context" - "flag" "fmt" "strings" "time" "github.com/davecgh/go-spew/spew" - "github.com/golang/glog" "go.opencensus.io/stats/view" "go.opencensus.io/trace" "go.uber.org/zap" @@ -35,9 +33,6 @@ import ( ) const ( - // VerboseLogLevel defines verbose log level as 10 - VerboseLogLevel glog.Level = 10 - // 1 second was chosen arbitrarily metricViewReportingPeriod = 1 * time.Second @@ -134,11 +129,6 @@ func InitializeMetricExporter(context string) { func InitializeLogger(logVerbose bool) { logLevel := "info" if logVerbose { - // Both gLog and "go test" use -v flag. The code below is a work around so that we can still set v value for gLog - flag.StringVar(&logLevel, "logLevel", fmt.Sprint(VerboseLogLevel), "verbose log level") - flag.Lookup("v").Value.Set(logLevel) - glog.Infof("Logging set to verbose mode with logLevel %d", VerboseLogLevel) - logLevel = "debug" } diff --git a/vendor/knative.dev/pkg/test/mako/alerter/alerter.go b/vendor/knative.dev/pkg/test/mako/alerter/alerter.go index 5852a946..1b15e69f 100644 --- a/vendor/knative.dev/pkg/test/mako/alerter/alerter.go +++ b/vendor/knative.dev/pkg/test/mako/alerter/alerter.go @@ -17,6 +17,9 @@ limitations under the License. package alerter import ( + "fmt" + "log" + qpb "github.com/google/mako/proto/quickstore/quickstore_go_proto" "knative.dev/pkg/test/helpers" "knative.dev/pkg/test/mako/alerter/github" @@ -31,38 +34,42 @@ type Alerter struct { } // SetupGitHub will setup SetupGitHub for the alerter. -func (alerter *Alerter) SetupGitHub(org, repo, githubTokenPath string) error { +func (alerter *Alerter) SetupGitHub(org, repo, githubTokenPath string) { issueHandler, err := github.Setup(org, repo, githubTokenPath, false) if err != nil { - return err + log.Printf("Error happens in setup '%v', Github alerter will not be enabled", err) } alerter.githubIssueHandler = issueHandler - return nil } // SetupSlack will setup Slack for the alerter. -func (alerter *Alerter) SetupSlack(userName, readTokenPath, writeTokenPath string, channels []config.Channel) error { +func (alerter *Alerter) SetupSlack(userName, readTokenPath, writeTokenPath string, channels []config.Channel) { messageHandler, err := slack.Setup(userName, readTokenPath, writeTokenPath, channels, false) if err != nil { - return err + log.Printf("Error happens in setup '%v', Slack alerter will not be enabled", err) } alerter.slackMessageHandler = messageHandler - return nil } // HandleBenchmarkResult will handle the benchmark result which returns from `q.Store()` -func (alerter *Alerter) HandleBenchmarkResult(testName string, output qpb.QuickstoreOutput, err error) error { +func (alerter *Alerter) HandleBenchmarkResult( + benchmarkKey, benchmarkName string, + output qpb.QuickstoreOutput, err error) error { if err != nil { if output.GetStatus() == qpb.QuickstoreOutput_ANALYSIS_FAIL { var errs []error - summary := output.GetSummaryOutput() + summary := fmt.Sprintf("%s\n\nSee run chart at: %s\n\nSee aggregate chart at: %s", + output.GetSummaryOutput(), + output.GetRunChartLink(), + fmt.Sprintf("mako.dev/benchmark?benchmark_key=%s&tseconds=604800", benchmarkKey), + ) if alerter.githubIssueHandler != nil { - if err := alerter.githubIssueHandler.CreateIssueForTest(testName, summary); err != nil { + if err := alerter.githubIssueHandler.CreateIssueForTest(benchmarkName, summary); err != nil { errs = append(errs, err) } } if alerter.slackMessageHandler != nil { - if err := alerter.slackMessageHandler.SendAlert(summary); err != nil { + if err := alerter.slackMessageHandler.SendAlert(benchmarkName, summary); err != nil { errs = append(errs, err) } } @@ -71,7 +78,7 @@ func (alerter *Alerter) HandleBenchmarkResult(testName string, output qpb.Quicks return err } if alerter.githubIssueHandler != nil { - return alerter.githubIssueHandler.CloseIssueForTest(testName) + return alerter.githubIssueHandler.CloseIssueForTest(benchmarkName) } return nil diff --git a/vendor/knative.dev/pkg/test/mako/alerter/github/issue.go b/vendor/knative.dev/pkg/test/mako/alerter/github/issue.go index e2d0faf1..6a1e2dc5 100644 --- a/vendor/knative.dev/pkg/test/mako/alerter/github/issue.go +++ b/vendor/knative.dev/pkg/test/mako/alerter/github/issue.go @@ -17,6 +17,7 @@ limitations under the License. package github import ( + "errors" "fmt" "time" @@ -28,8 +29,14 @@ import ( const ( // perfLabel is the Github issue label used for querying all auto-generated performance issues. - perfLabel = "auto:perf" - daysConsideredOld = 10 // arbitrary number of days for an issue to be considered old + perfLabel = "auto:perf" + // number of days for an issue to be considered old + daysConsideredOld = 30 + + // number of days for an issue to be considered active + // To avoid frequent open/close actions, only automatically close an issue if there is no activity + // (update, comment, etc.) on it for a specified time + daysConsideredActive = 3 // issueTitleTemplate is a template for issue title issueTitleTemplate = "[performance] %s" @@ -37,10 +44,11 @@ const ( // issueBodyTemplate is a template for issue body issueBodyTemplate = ` ### Auto-generated issue tracking performance regression -* **Test name**: %s` +* **Test name**: %s +* **Repository name**: %s` - // newIssueCommentTemplate is a template for the comment of an issue that has been quiet for a long time - newIssueCommentTemplate = ` + // issueSummaryCommentTemplate is a template for the summary of an issue + issueSummaryCommentTemplate = ` A new regression for this test has been detected: %s` @@ -49,9 +57,9 @@ A new regression for this test has been detected: New regression has been detected, reopening this issue: %s` - // closeIssueComment is the comment of an issue when we close it + // closeIssueComment is the comment of an issue when it is closed closeIssueComment = ` -The performance regression goes way for this test, closing this issue.` +The performance regression goes away for this test, closing this issue.` ) // IssueHandler handles methods for github issues @@ -69,6 +77,12 @@ type config struct { // Setup creates the necessary setup to make calls to work with github issues func Setup(org, repo, githubTokenPath string, dryrun bool) (*IssueHandler, error) { + if org == "" { + return nil, errors.New("org cannot be empty") + } + if repo == "" { + return nil, errors.New("repo cannot be empty") + } ghc, err := ghutil.NewGithubClient(githubTokenPath) if err != nil { return nil, fmt.Errorf("cannot authenticate to github: %v", err) @@ -80,65 +94,75 @@ func Setup(org, repo, githubTokenPath string, dryrun bool) (*IssueHandler, error // CreateIssueForTest will try to add an issue with the given testName and description. // If there is already an issue related to the test, it will try to update that issue. func (gih *IssueHandler) CreateIssueForTest(testName, desc string) error { - org := gih.config.org - repo := gih.config.repo - dryrun := gih.config.dryrun title := fmt.Sprintf(issueTitleTemplate, testName) - issue := gih.findIssue(org, repo, title, dryrun) + issue, err := gih.findIssue(title) + if err != nil { + return fmt.Errorf("failed to find issues for test %q: %v, skipped creating new issue", testName, err) + } // If the issue hasn't been created, create one if issue == nil { - body := fmt.Sprintf(issueBodyTemplate, testName) - issue, err := gih.createNewIssue(org, repo, title, body, dryrun) + commentBody := fmt.Sprintf(issueBodyTemplate, testName, gih.config.repo) + issue, err := gih.createNewIssue(title, commentBody) if err != nil { - return err + return fmt.Errorf("failed to create a new issue for test %q: %v", testName, err) } - comment := fmt.Sprintf(newIssueCommentTemplate, desc) - if err := gih.addComment(org, repo, *issue.Number, comment, dryrun); err != nil { - return err + commentBody = fmt.Sprintf(issueSummaryCommentTemplate, desc) + if err := gih.addComment(*issue.Number, commentBody); err != nil { + return fmt.Errorf("failed to add comment for new issue %d: %v", *issue.Number, err) } - // If one issue with the same title has been closed, reopen it and add new comment - } else if *issue.State == string(ghutil.IssueCloseState) { - if err := gih.reopenIssue(org, repo, *issue.Number, dryrun); err != nil { - return err + return nil + } + + // If the issue has been created, edit it + issueNumber := *issue.Number + + // If the issue has been closed, reopen it + if *issue.State == string(ghutil.IssueCloseState) { + if err := gih.reopenIssue(issueNumber); err != nil { + return fmt.Errorf("failed to reopen issue %d: %v", issueNumber, err) } - comment := fmt.Sprintf(reopenIssueCommentTemplate, desc) - if err := gih.addComment(org, repo, *issue.Number, comment, dryrun); err != nil { - return err - } - } else { - // If the issue hasn't been updated for a long time, add a new comment - if time.Now().Sub(*issue.UpdatedAt) > daysConsideredOld*24*time.Hour { - comment := fmt.Sprintf(newIssueCommentTemplate, desc) - // TODO(Fredy-Z): edit the old comment instead of adding a new one, like flaky-test-reporter - if err := gih.addComment(org, repo, *issue.Number, comment, dryrun); err != nil { - return err - } + commentBody := fmt.Sprintf(reopenIssueCommentTemplate, desc) + if err := gih.addComment(issueNumber, commentBody); err != nil { + return fmt.Errorf("failed to add comment for reopened issue %d: %v", issueNumber, err) } } + // Edit the old comment + comments, err := gih.getComments(issueNumber) + if err != nil { + return fmt.Errorf("failed to get comments from issue %d: %v", issueNumber, err) + } + if len(comments) < 2 { + return fmt.Errorf("existing issue %d is malformed, cannot update", issueNumber) + } + commentBody := fmt.Sprintf(issueSummaryCommentTemplate, desc) + if err := gih.editComment(issueNumber, *comments[1].ID, commentBody); err != nil { + return fmt.Errorf("failed to edit the comment for issue %d: %v", issueNumber, err) + } + return nil } // createNewIssue will create a new issue, and add perfLabel for it. -func (gih *IssueHandler) createNewIssue(org, repo, title, body string, dryrun bool) (*github.Issue, error) { +func (gih *IssueHandler) createNewIssue(title, body string) (*github.Issue, error) { var newIssue *github.Issue if err := helpers.Run( - "creating issue", + fmt.Sprintf("creating issue %q in %q", title, gih.config.repo), func() error { var err error - newIssue, err = gih.client.CreateIssue(org, repo, title, body) + newIssue, err = gih.client.CreateIssue(gih.config.org, gih.config.repo, title, body) return err }, - dryrun, + gih.config.dryrun, ); nil != err { return nil, err } if err := helpers.Run( - "adding perf label", + fmt.Sprintf("adding perf label for issue %q in %q", title, gih.config.repo), func() error { - return gih.client.AddLabelsToIssue(org, repo, *newIssue.Number, []string{perfLabel}) + return gih.client.AddLabelsToIssue(gih.config.org, gih.config.repo, *newIssue.Number, []string{perfLabel}) }, - dryrun, + gih.config.dryrun, ); nil != err { return nil, err } @@ -148,74 +172,119 @@ func (gih *IssueHandler) createNewIssue(org, repo, title, body string, dryrun bo // CloseIssueForTest will try to close the issue for the given testName. // If there is no issue related to the test or the issue is already closed, the function will do nothing. func (gih *IssueHandler) CloseIssueForTest(testName string) error { - org := gih.config.org - repo := gih.config.repo - dryrun := gih.config.dryrun title := fmt.Sprintf(issueTitleTemplate, testName) - issue := gih.findIssue(org, repo, title, dryrun) - if issue == nil || *issue.State == string(ghutil.IssueCloseState) { + issue, err := gih.findIssue(title) + // If no issue has been found, or the issue has already been closed, do nothing. + if issue == nil || err != nil || *issue.State == string(ghutil.IssueCloseState) { + return nil + } + // If the issue is still active, do not close it. + if time.Now().Sub(issue.GetUpdatedAt()) < daysConsideredActive*24*time.Hour { return nil } issueNumber := *issue.Number - if err := helpers.Run( - "add comment for the issue to close", - func() error { - _, cErr := gih.client.CreateComment(org, repo, issueNumber, closeIssueComment) - return cErr - }, - dryrun, - ); err != nil { - return err + if err := gih.addComment(issueNumber, closeIssueComment); err != nil { + return fmt.Errorf("failed to add comment for the issue %d to close: %v", issueNumber, err) } - return helpers.Run( - "closing issue", - func() error { - return gih.client.CloseIssue(org, repo, issueNumber) - }, - dryrun, - ) -} - -// reopenIssue will reopen the given issue. -func (gih *IssueHandler) reopenIssue(org, repo string, issueNumber int, dryrun bool) error { - return helpers.Run( - "reopen the issue", - func() error { - return gih.client.ReopenIssue(org, repo, issueNumber) - }, - dryrun, - ) -} - -// findIssue will return the issue in the given repo if it exists. -func (gih *IssueHandler) findIssue(org, repo, title string, dryrun bool) *github.Issue { - var issues []*github.Issue - helpers.Run( - "list issues in the repo", - func() error { - var err error - issues, err = gih.client.ListIssuesByRepo(org, repo, []string{perfLabel}) - return err - }, - dryrun, - ) - for _, issue := range issues { - if *issue.Title == title { - return issue - } + if err := gih.closeIssue(issueNumber); err != nil { + return fmt.Errorf("failed to close the issue %d: %v", issueNumber, err) } return nil } -// addComment will add comment for the given issue. -func (gih *IssueHandler) addComment(org, repo string, issueNumber int, commentBody string, dryrun bool) error { +// reopenIssue will reopen the given issue. +func (gih *IssueHandler) reopenIssue(issueNumber int) error { return helpers.Run( - "add comment for issue", + fmt.Sprintf("reopening issue %d in %q", issueNumber, gih.config.repo), func() error { - _, err := gih.client.CreateComment(org, repo, issueNumber, commentBody) - return err + return gih.client.ReopenIssue(gih.config.org, gih.config.repo, issueNumber) }, - dryrun, + gih.config.dryrun, + ) +} + +// closeIssue will close the given issue. +func (gih *IssueHandler) closeIssue(issueNumber int) error { + return helpers.Run( + fmt.Sprintf("closing issue %d in %q", issueNumber, gih.config.repo), + func() error { + return gih.client.CloseIssue(gih.config.org, gih.config.repo, issueNumber) + }, + gih.config.dryrun, + ) +} + +// findIssue will return the issue in the given repo if it exists. +func (gih *IssueHandler) findIssue(title string) (*github.Issue, error) { + var issues []*github.Issue + if err := helpers.Run( + fmt.Sprintf("listing issues in %q", gih.config.repo), + func() error { + var err error + issues, err = gih.client.ListIssuesByRepo(gih.config.org, gih.config.repo, []string{perfLabel}) + return err + }, + gih.config.dryrun, + ); err != nil { + return nil, err + } + + var existingIssue *github.Issue + for _, issue := range issues { + if *issue.Title == title { + // If the issue has been closed a long time ago, ignore this issue. + if issue.GetState() == string(ghutil.IssueCloseState) && + time.Now().Sub(*issue.UpdatedAt) > daysConsideredOld*24*time.Hour { + continue + } + + // If there are multiple issues, return the one that was created most recently. + if existingIssue == nil || issue.CreatedAt.After(*existingIssue.CreatedAt) { + existingIssue = issue + } + } + } + + return existingIssue, nil +} + +// getComments will get comments for the given issue. +func (gih *IssueHandler) getComments(issueNumber int) ([]*github.IssueComment, error) { + var comments []*github.IssueComment + if err := helpers.Run( + fmt.Sprintf("getting comments for issue %d in %q", issueNumber, gih.config.repo), + func() error { + var err error + comments, err = gih.client.ListComments(gih.config.org, gih.config.repo, issueNumber) + return err + }, + gih.config.dryrun, + ); err != nil { + return comments, err + } + return comments, nil +} + +// addComment will add comment for the given issue. +func (gih *IssueHandler) addComment(issueNumber int, commentBody string) error { + return helpers.Run( + fmt.Sprintf("adding comment %q for issue %d in %q", commentBody, issueNumber, gih.config.repo), + func() error { + _, err := gih.client.CreateComment(gih.config.org, gih.config.repo, issueNumber, commentBody) + return err + }, + gih.config.dryrun, + ) +} + +// editComment will edit the comment to the new body. +func (gih *IssueHandler) editComment(issueNumber int, commentID int64, commentBody string) error { + return helpers.Run( + fmt.Sprintf("editting comment to %q for issue %d in %q", commentBody, issueNumber, gih.config.repo), + func() error { + return gih.client.EditComment(gih.config.org, gih.config.repo, commentID, commentBody) + }, + gih.config.dryrun, ) } diff --git a/vendor/knative.dev/pkg/test/mako/alerter/slack/message.go b/vendor/knative.dev/pkg/test/mako/alerter/slack/message.go index 264eea8f..9e07ac97 100644 --- a/vendor/knative.dev/pkg/test/mako/alerter/slack/message.go +++ b/vendor/knative.dev/pkg/test/mako/alerter/slack/message.go @@ -31,7 +31,7 @@ var minInterval = flag.Duration("min-alert-interval", 24*time.Hour, "The minimum const ( messageTemplate = ` -As of %s, there is a new performance regression detected from automation test: +As of %s, there is a new performance regression detected from test automation for **%s**: %s` ) @@ -61,8 +61,8 @@ func Setup(userName, readTokenPath, writeTokenPath string, channels []config.Cha }, nil } -// SendAlert will send the alert text to the slack channel(s) -func (smh *MessageHandler) SendAlert(text string) error { +// SendAlert will send alert for performance regression to the slack channel(s) +func (smh *MessageHandler) SendAlert(testName, summary string) error { dryrun := smh.dryrun errCh := make(chan error) var wg sync.WaitGroup @@ -90,7 +90,7 @@ func (smh *MessageHandler) SendAlert(text string) error { return } // send the alert message to the channel - message := fmt.Sprintf(messageTemplate, time.Now(), text) + message := fmt.Sprintf(messageTemplate, time.Now().UTC(), testName, summary) if err := helpers.Run( fmt.Sprintf("sending message %q to channel %q", message, channel.Name), func() error { diff --git a/vendor/knative.dev/pkg/test/mako/config/configmap.go b/vendor/knative.dev/pkg/test/mako/config/configmap.go index 49fea59b..8fb48e19 100644 --- a/vendor/knative.dev/pkg/test/mako/config/configmap.go +++ b/vendor/knative.dev/pkg/test/mako/config/configmap.go @@ -31,6 +31,9 @@ const ( // Config defines the mako configuration options. type Config struct { + // Organization holds the name of the organization for the current repository. + Organization string + // Repository holds the name of the repository that runs the benchmarks. Repository string @@ -53,6 +56,9 @@ func NewConfigFromMap(data map[string]string) (*Config, error) { AdditionalTags: []string{}, } + if raw, ok := data["organization"]; ok { + lc.Organization = raw + } if raw, ok := data["repository"]; ok { lc.Repository = raw } diff --git a/vendor/knative.dev/pkg/test/mako/config/environment.go b/vendor/knative.dev/pkg/test/mako/config/environment.go index d65a6a5f..62e72c24 100644 --- a/vendor/knative.dev/pkg/test/mako/config/environment.go +++ b/vendor/knative.dev/pkg/test/mako/config/environment.go @@ -22,14 +22,27 @@ import ( // TODO: perhaps cache the loaded CM. -// MustGetRepository returns the repository from the configmap, or dies. -func MustGetRepository() string { +const defaultOrg = "knative" + +// GetOrganization returns the organization from the configmap. +// It will return the defaultOrg if any error happens or it's empty. +func GetOrganization() string { cfg, err := loadConfig() if err != nil { - log.Fatalf("unable to load config from the configmap: %v", err) + return defaultOrg } - if cfg.Repository == "" { - log.Fatal("unable to get repository from the configmap") + if cfg.Organization == "" { + return defaultOrg + } + return cfg.Organization +} + +// GetRepository returns the repository from the configmap. +// It will return an empty string if any error happens. +func GetRepository() string { + cfg, err := loadConfig() + if err != nil { + return "" } return cfg.Repository } diff --git a/vendor/knative.dev/pkg/test/mako/sidecar.go b/vendor/knative.dev/pkg/test/mako/sidecar.go index fc606e5e..add00c50 100644 --- a/vendor/knative.dev/pkg/test/mako/sidecar.go +++ b/vendor/knative.dev/pkg/test/mako/sidecar.go @@ -49,13 +49,24 @@ const ( // slackUserName is the slack user name that is used by Slack client slackUserName = "Knative Testgrid Robot" + + // These token settings are for alerter. + // If we want to enable the alerter for a benchmark, we need to mount the + // token to the pod, with the same name and path. + // See https://github.com/knative/serving/blob/master/test/performance/dataplane-probe/dataplane-probe.yaml + tokenFolder = "/var/secret" + githubToken = "github-token" + slackReadToken = "slack-read-token" + slackWriteToken = "slack-write-token" ) // Client is a wrapper that wraps all Mako related operations type Client struct { - Quickstore *quickstore.Quickstore - Context context.Context - ShutDownFunc func(context.Context) + Quickstore *quickstore.Quickstore + Context context.Context + ShutDownFunc func(context.Context) + + benchmarkKey string benchmarkName string alerter *alerter.Alerter } @@ -63,7 +74,7 @@ type Client struct { // StoreAndHandleResult stores the benchmarking data and handles the result. func (c *Client) StoreAndHandleResult() error { out, err := c.Quickstore.Store() - return c.alerter.HandleBenchmarkResult(c.benchmarkName, out, err) + return c.alerter.HandleBenchmarkResult(c.benchmarkKey, c.benchmarkName, out, err) } // EscapeTag replaces characters that Mako doesn't accept with ones it does. @@ -71,11 +82,11 @@ func EscapeTag(tag string) string { return strings.ReplaceAll(tag, ".", "_") } -// Setup sets up the mako client for the provided benchmarkKey. +// SetupHelper sets up the mako client for the provided benchmarkKey. // It will add a few common tags and allow each benchmark to add custm tags as well. // It returns the mako client handle to store metrics, a method to close the connection // to mako server once done and error in case of failures. -func Setup(ctx context.Context, extraTags ...string) (*Client, error) { +func SetupHelper(ctx context.Context, benchmarkKey *string, benchmarkName *string, extraTags ...string) (*Client, error) { tags := append(config.MustGetTags(), extraTags...) // Get the commit of the benchmarks commitID, err := changeset.Get() @@ -125,7 +136,6 @@ func Setup(ctx context.Context, extraTags ...string) (*Client, error) { tags = append(tags, "instanceType="+EscapeTag(parts[3])) } - benchmarkKey, benchmarkName := config.MustGetBenchmark() // Create a new Quickstore that connects to the microservice qs, qclose, err := quickstore.NewAtAddress(ctx, &qpb.QuickstoreInput{ BenchmarkKey: benchmarkKey, @@ -143,13 +153,13 @@ func Setup(ctx context.Context, extraTags ...string) (*Client, error) { alerter := &alerter.Alerter{} alerter.SetupGitHub( org, - config.MustGetRepository(), - tokenPath("github-token"), + config.GetRepository(), + tokenPath(githubToken), ) alerter.SetupSlack( slackUserName, - tokenPath("slack-read-token"), - tokenPath("slack-write-token"), + tokenPath(slackReadToken), + tokenPath(slackWriteToken), config.GetSlackChannels(*benchmarkName), ) @@ -158,12 +168,22 @@ func Setup(ctx context.Context, extraTags ...string) (*Client, error) { Context: ctx, ShutDownFunc: qclose, alerter: alerter, + benchmarkKey: *benchmarkKey, benchmarkName: *benchmarkName, } return client, nil } -func tokenPath(token string) string { - return filepath.Join("/var/secrets", token) +func Setup(ctx context.Context, extraTags ...string) (*Client, error) { + benchmarkKey, benchmarkName := config.MustGetBenchmark() + return SetupHelper(ctx, benchmarkKey, benchmarkName, extraTags...) +} + +func SetupWithBenchmarkConfig(ctx context.Context, benchmarkKey *string, benchmarkName *string, extraTags ...string) (*Client, error) { + return SetupHelper(ctx, benchmarkKey, benchmarkName, extraTags...) +} + +func tokenPath(token string) string { + return filepath.Join(tokenFolder, token) } diff --git a/vendor/knative.dev/pkg/test/prow/prow.go b/vendor/knative.dev/pkg/test/prow/prow.go new file mode 100644 index 00000000..957ed52a --- /dev/null +++ b/vendor/knative.dev/pkg/test/prow/prow.go @@ -0,0 +1,335 @@ +/* +Copyright 2019 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// prow.go defines types and functions specific to prow logics +// All paths used in this package are gcs paths unless specified otherwise + +package prow + +import ( + "bufio" + "context" + "encoding/json" + "log" + "os" + "path" + "sort" + "strconv" + "strings" + + "knative.dev/pkg/test/gcs" +) + +const ( + // OrgName is the name of knative org + OrgName = "knative" + + // BucketName is the gcs bucket for all knative builds + BucketName = "knative-prow" + // Latest is the filename storing latest build number + Latest = "latest-build.txt" + // BuildLog is the filename for build log + BuildLog = "build-log.txt" + // StartedJSON is the json file containing build started info + StartedJSON = "started.json" + // FinishedJSON is the json file containing build finished info + FinishedJSON = "finished.json" + // ArtifactsDir is the dir containing artifacts + ArtifactsDir = "artifacts" + + // PresubmitJob means it runs on unmerged PRs. + PresubmitJob = "presubmit" + // PostsubmitJob means it runs on each new commit. + PostsubmitJob = "postsubmit" + // PeriodicJob means it runs on a time-basis, unrelated to git changes. + PeriodicJob = "periodic" + // BatchJob tests multiple unmerged PRs at the same time. + BatchJob = "batch" +) + +// defined here so that it can be mocked for unit testing +var logFatalf = log.Fatalf + +var ctx = context.Background() + +// Job struct represents a job directory in gcs. +// gcs job StoragePath will be derived from Type if it's defined, +type Job struct { + Name string + Type string + Bucket string // optional + Repo string // optional + StoragePath string // optional + PullID int // only for Presubmit jobs + Builds []Build // optional +} + +// Build points to a build stored under a particular gcs path. +type Build struct { + JobName string + StoragePath string + BuildID int + Bucket string // optional + StartTime *int64 + FinishTime *int64 +} + +// Started holds the started.json values of the build. +type Started struct { + Timestamp int64 `json:"timestamp"` // epoch seconds + RepoVersion string `json:"repo-version"` + Node string `json:"node"` + Pull string `json:"pull"` + Repos map[string]string `json:"repos"` // {repo: branch_or_pull} map +} + +// Finished holds the finished.json values of the build +type Finished struct { + // Timestamp is epoch seconds + Timestamp int64 `json:"timestamp"` + Passed bool `json:"passed"` + JobVersion string `json:"job-version"` + Metadata Metadata `json:"metadata"` +} + +// Metadata contains metadata in finished.json +type Metadata map[string]interface{} + +/* Local logics */ + +// GetLocalArtifactsDir gets the artifacts directory where prow looks for artifacts. +// By default, it will look at the env var ARTIFACTS. +func GetLocalArtifactsDir() string { + dir := os.Getenv("ARTIFACTS") + if dir == "" { + log.Printf("Env variable ARTIFACTS not set. Using %s instead.", ArtifactsDir) + dir = ArtifactsDir + } + return dir +} + +/* GCS related logics */ + +// Initialize wraps gcs authentication, have to be invoked before any other functions +func Initialize(serviceAccount string) error { + return gcs.Authenticate(ctx, serviceAccount) +} + +// NewJob creates new job struct +// pullID is only saved by Presubmit job for determining StoragePath +func NewJob(jobName, jobType, repoName string, pullID int) *Job { + job := Job{ + Name: jobName, + Type: jobType, + Bucket: BucketName, + } + + switch jobType { + case PeriodicJob, PostsubmitJob: + job.StoragePath = path.Join("logs", jobName) + case PresubmitJob: + job.PullID = pullID + job.StoragePath = path.Join("pr-logs", "pull", OrgName+"_"+repoName, strconv.Itoa(pullID), jobName) + case BatchJob: + job.StoragePath = path.Join("pr-logs", "pull", "batch", jobName) + default: + logFatalf("unknown job spec type: %v", jobType) + } + return &job +} + +// PathExists checks if the storage path of a job exists in gcs or not +func (j *Job) PathExists() bool { + return gcs.Exists(ctx, BucketName, j.StoragePath) +} + +// GetLatestBuildNumber gets the latest build number for job +func (j *Job) GetLatestBuildNumber() (int, error) { + logFilePath := path.Join(j.StoragePath, Latest) + contents, err := gcs.Read(ctx, BucketName, logFilePath) + if err != nil { + return 0, err + } + latestBuild, err := strconv.Atoi(strings.TrimSuffix(string(contents), "\n")) + if err != nil { + return 0, err + } + return latestBuild, nil +} + +// NewBuild gets build struct based on job info +// No gcs operation is performed by this function +func (j *Job) NewBuild(buildID int) *Build { + build := Build{ + Bucket: BucketName, + JobName: j.Name, + StoragePath: path.Join(j.StoragePath, strconv.Itoa(buildID)), + BuildID: buildID, + } + + if startTime, err := build.GetStartTime(); err == nil { + build.StartTime = &startTime + } + if finishTime, err := build.GetFinishTime(); err == nil { + build.FinishTime = &finishTime + } + return &build +} + +// GetFinishedBuilds gets all builds that have finished, +// by looking at existence of "finished.json" file +func (j *Job) GetFinishedBuilds() []Build { + var finishedBuilds []Build + builds := j.GetBuilds() + for _, build := range builds { + if build.IsFinished() { + finishedBuilds = append(finishedBuilds, build) + } + } + return finishedBuilds +} + +// GetBuilds gets all builds from this job on gcs, precomputes start/finish time of builds +// by parsing "Started.json" and "Finished.json" on gcs, could be very expensive if there are +// large number of builds +func (j *Job) GetBuilds() []Build { + var builds []Build + for _, ID := range j.GetBuildIDs() { + builds = append(builds, *j.NewBuild(ID)) + } + return builds +} + +// GetBuildIDs gets all build IDs from this job on gcs, scans all direct child of gcs directory +// for job, keeps the ones that can be parsed as integer +func (j *Job) GetBuildIDs() []int { + var buildIDs []int + gcsBuildPaths := gcs.ListDirectChildren(ctx, j.Bucket, j.StoragePath) + for _, gcsBuildPath := range gcsBuildPaths { + if buildID, err := getBuildIDFromBuildPath(gcsBuildPath); err == nil { + buildIDs = append(buildIDs, buildID) + } + } + return buildIDs +} + +// GetLatestBuilds get latest builds from gcs, sort by start time from newest to oldest, +// will return count number of builds +func (j *Job) GetLatestBuilds(count int) []Build { + // The timestamp of gcs directories are not usable, + // as they are all set to '0001-01-01 00:00:00 +0000 UTC', + // so use 'started.json' creation date for latest builds + builds := j.GetFinishedBuilds() + sort.Slice(builds, func(i, j int) bool { + if builds[i].StartTime == nil { + return false + } + if builds[j].StartTime == nil { + return true + } + return *builds[i].StartTime > *builds[j].StartTime + }) + if len(builds) < count { + return builds + } + return builds[:count] +} + +// IsStarted check if build has started by looking at "started.json" file +func (b *Build) IsStarted() bool { + return gcs.Exists(ctx, BucketName, path.Join(b.StoragePath, StartedJSON)) +} + +// IsFinished check if build has finished by looking at "finished.json" file +func (b *Build) IsFinished() bool { + return gcs.Exists(ctx, BucketName, path.Join(b.StoragePath, FinishedJSON)) +} + +// GetStartTime gets started timestamp of a build, +// returning -1 if the build didn't start or if it failed to get the timestamp +func (b *Build) GetStartTime() (int64, error) { + var started Started + if err := unmarshalJSONFile(path.Join(b.StoragePath, StartedJSON), &started); err != nil { + return -1, err + } + return started.Timestamp, nil +} + +// GetFinishTime gets finished timestamp of a build, +// returning -1 if the build didn't finish or if it failed to get the timestamp +func (b *Build) GetFinishTime() (int64, error) { + var finished Finished + if err := unmarshalJSONFile(path.Join(b.StoragePath, FinishedJSON), &finished); err != nil { + return -1, err + } + return finished.Timestamp, nil +} + +// GetArtifacts gets gcs path for all artifacts of current build +func (b *Build) GetArtifacts() []string { + return gcs.ListChildrenFiles(ctx, BucketName, b.GetArtifactsDir()) +} + +// GetArtifactsDir gets gcs path for artifacts of current build +func (b *Build) GetArtifactsDir() string { + return path.Join(b.StoragePath, ArtifactsDir) +} + +// GetBuildLogPath gets "build-log.txt" path for current build +func (b *Build) GetBuildLogPath() string { + return path.Join(b.StoragePath, BuildLog) +} + +// ReadFile reads given file of current build, +// relPath is the file path relative to build directory +func (b *Build) ReadFile(relPath string) ([]byte, error) { + return gcs.Read(ctx, BucketName, path.Join(b.StoragePath, relPath)) +} + +// ParseLog parses the build log and returns the lines where the checkLog func does not return an empty slice, +// checkLog function should take in the log statement and return a part from that statement that should be in the log output. +func (b *Build) ParseLog(checkLog func(s []string) *string) ([]string, error) { + var logs []string + + f, err := gcs.NewReader(ctx, b.Bucket, b.GetBuildLogPath()) + if err != nil { + return logs, err + } + defer f.Close() + scanner := bufio.NewScanner(f) + for scanner.Scan() { + if s := checkLog(strings.Fields(scanner.Text())); s != nil { + logs = append(logs, *s) + } + } + return logs, nil +} + +// getBuildIDFromBuildPath digests gcs build path and return last portion of path +func getBuildIDFromBuildPath(buildPath string) (int, error) { + _, buildIDStr := path.Split(strings.TrimRight(buildPath, " /")) + return strconv.Atoi(buildIDStr) +} + +// unmarshalJSONFile reads a file from gcs, parses it with xml and write to v. +// v must be an arbitrary struct, slice, or string. +func unmarshalJSONFile(storagePath string, v interface{}) error { + contents, err := gcs.Read(ctx, BucketName, storagePath) + if err != nil { + return err + } + return json.Unmarshal(contents, v) +} diff --git a/vendor/knative.dev/pkg/test/spoof/error_checks.go b/vendor/knative.dev/pkg/test/spoof/error_checks.go index 0cd2995c..8a913b99 100644 --- a/vendor/knative.dev/pkg/test/spoof/error_checks.go +++ b/vendor/knative.dev/pkg/test/spoof/error_checks.go @@ -40,13 +40,14 @@ func isDNSError(err error) bool { return strings.Contains(msg, "no such host") || strings.Contains(msg, ":53") } -func isTCPConnectRefuse(err error) bool { +func isConnectionRefused(err error) bool { // The alternative for the string check is: // errNo := (((err.(*url.Error)).Err.(*net.OpError)).Err.(*os.SyscallError).Err).(syscall.Errno) // if errNo == syscall.Errno(0x6f) {...} // But with assertions, of course. - if err != nil && strings.Contains(err.Error(), "connect: connection refused") { - return true - } - return false + return err != nil && strings.Contains(err.Error(), "connect: connection refused") +} + +func isConnectionReset(err error) bool { + return err != nil && strings.Contains(err.Error(), "connection reset by peer") } diff --git a/vendor/knative.dev/pkg/test/spoof/spoof.go b/vendor/knative.dev/pkg/test/spoof/spoof.go index 6640e54f..f7ac6a51 100644 --- a/vendor/knative.dev/pkg/test/spoof/spoof.go +++ b/vendor/knative.dev/pkg/test/spoof/spoof.go @@ -107,7 +107,7 @@ func New( opts ...TransportOption) (*SpoofingClient, error) { endpoint, err := ResolveEndpoint(kubeClientset, domain, resolvable, endpointOverride) if err != nil { - fmt.Errorf("failed get the cluster endpoint: %v", err) + return nil, fmt.Errorf("failed get the cluster endpoint: %v", err) } // Spoof the hostname at the resolver level @@ -214,19 +214,22 @@ func (sc *SpoofingClient) Poll(req *http.Request, inState ResponseChecker) (*Res resp, err = sc.Do(req) if err != nil { if isTCPTimeout(err) { - sc.Logf("Retrying %s for TCP timeout %v", req.URL.String(), err) + sc.Logf("Retrying %s for TCP timeout: %v", req.URL, err) return false, nil } // Retrying on DNS error, since we may be using xip.io or nip.io in tests. if isDNSError(err) { - sc.Logf("Retrying %s for DNS error %v", req.URL.String(), err) + sc.Logf("Retrying %s for DNS error: %v", req.URL, err) return false, nil } // Repeat the poll on `connection refused` errors, which are usually transient Istio errors. - if isTCPConnectRefuse(err) { - sc.Logf("Retrying %s for connection refused %v", req.URL.String(), err) + if isConnectionRefused(err) { + sc.Logf("Retrying %s for connection refused: %v", req.URL, err) return false, nil } + if isConnectionReset(err) { + sc.Logf("Retrying %s for connection reset: %v", req.URL, err) + } return true, err } diff --git a/vendor/knative.dev/pkg/test/zipkin/util.go b/vendor/knative.dev/pkg/test/zipkin/util.go index fc13b221..a1b6f635 100644 --- a/vendor/knative.dev/pkg/test/zipkin/util.go +++ b/vendor/knative.dev/pkg/test/zipkin/util.go @@ -20,6 +20,7 @@ package zipkin import ( "encoding/json" + "fmt" "io/ioutil" "net/http" "sync" @@ -134,26 +135,29 @@ func JSONTrace(traceID string, expected int, timeout time.Duration) (trace []mod for len(trace) != expected { select { case <-t: - return trace, &TimeoutError{} + return trace, &TimeoutError{ + lastErr: err, + } default: trace, err = jsonTrace(traceID) - if err != nil { - return trace, err - } } } - return trace, nil + return trace, err } // TimeoutError is an error returned by JSONTrace if it times out before getting the expected number // of traces. -type TimeoutError struct{} - -func (*TimeoutError) Error() string { - return "timeout getting JSONTrace" +type TimeoutError struct{ + lastErr error } -// jsonTrace gets a trace from Zipkin and returns it. +func (t *TimeoutError) Error() string { + return fmt.Sprintf("timeout getting JSONTrace, most recent error: %v", t.lastErr) +} + +// jsonTrace gets a trace from Zipkin and returns it. Errors returned from this function should be +// retried, as they are likely caused by random problems communicating with Zipkin, or Zipkin +// communicating with its data store. func jsonTrace(traceID string) ([]model.SpanModel, error) { var empty []model.SpanModel @@ -171,7 +175,7 @@ func jsonTrace(traceID string) ([]model.SpanModel, error) { var models []model.SpanModel err = json.Unmarshal(body, &models) if err != nil { - return empty, err + return empty, fmt.Errorf("got an error in unmarshalling JSON %q: %v", body, err) } return models, nil } diff --git a/vendor/knative.dev/pkg/testutils/clustermanager/boskos/boskos.go b/vendor/knative.dev/pkg/testutils/clustermanager/e2e-tests/boskos/boskos.go similarity index 97% rename from vendor/knative.dev/pkg/testutils/clustermanager/boskos/boskos.go rename to vendor/knative.dev/pkg/testutils/clustermanager/e2e-tests/boskos/boskos.go index 612dd5fb..81c5213a 100644 --- a/vendor/knative.dev/pkg/testutils/clustermanager/boskos/boskos.go +++ b/vendor/knative.dev/pkg/testutils/clustermanager/e2e-tests/boskos/boskos.go @@ -21,7 +21,7 @@ import ( "fmt" "time" - "knative.dev/pkg/testutils/common" + "knative.dev/pkg/testutils/clustermanager/e2e-tests/common" boskosclient "k8s.io/test-infra/boskos/client" boskoscommon "k8s.io/test-infra/boskos/common" diff --git a/vendor/knative.dev/pkg/testutils/clustermanager/boskos/fake/fake.go b/vendor/knative.dev/pkg/testutils/clustermanager/e2e-tests/boskos/fake/fake.go similarity index 97% rename from vendor/knative.dev/pkg/testutils/clustermanager/boskos/fake/fake.go rename to vendor/knative.dev/pkg/testutils/clustermanager/e2e-tests/boskos/fake/fake.go index f9154c45..d2a2cedd 100644 --- a/vendor/knative.dev/pkg/testutils/clustermanager/boskos/fake/fake.go +++ b/vendor/knative.dev/pkg/testutils/clustermanager/e2e-tests/boskos/fake/fake.go @@ -20,7 +20,7 @@ import ( "fmt" boskoscommon "k8s.io/test-infra/boskos/common" - "knative.dev/pkg/testutils/clustermanager/boskos" + "knative.dev/pkg/testutils/clustermanager/e2e-tests/boskos" ) const ( diff --git a/vendor/knative.dev/pkg/testutils/clustermanager/client.go b/vendor/knative.dev/pkg/testutils/clustermanager/e2e-tests/client.go similarity index 100% rename from vendor/knative.dev/pkg/testutils/clustermanager/client.go rename to vendor/knative.dev/pkg/testutils/clustermanager/e2e-tests/client.go diff --git a/vendor/knative.dev/pkg/testutils/common/common.go b/vendor/knative.dev/pkg/testutils/clustermanager/e2e-tests/common/common.go similarity index 100% rename from vendor/knative.dev/pkg/testutils/common/common.go rename to vendor/knative.dev/pkg/testutils/clustermanager/e2e-tests/common/common.go diff --git a/vendor/knative.dev/pkg/testutils/clustermanager/doc.go b/vendor/knative.dev/pkg/testutils/clustermanager/e2e-tests/doc.go similarity index 100% rename from vendor/knative.dev/pkg/testutils/clustermanager/doc.go rename to vendor/knative.dev/pkg/testutils/clustermanager/e2e-tests/doc.go diff --git a/vendor/knative.dev/pkg/testutils/clustermanager/e2e-tests/gke.go b/vendor/knative.dev/pkg/testutils/clustermanager/e2e-tests/gke.go new file mode 100644 index 00000000..d24ea1df --- /dev/null +++ b/vendor/knative.dev/pkg/testutils/clustermanager/e2e-tests/gke.go @@ -0,0 +1,358 @@ +/* +Copyright 2019 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package clustermanager + +import ( + "errors" + "fmt" + "log" + "strings" + + container "google.golang.org/api/container/v1beta1" + "knative.dev/pkg/test/gke" + "knative.dev/pkg/testutils/clustermanager/e2e-tests/boskos" + "knative.dev/pkg/testutils/clustermanager/e2e-tests/common" +) + +const ( + DefaultGKEMinNodes = 1 + DefaultGKEMaxNodes = 3 + DefaultGKENodeType = "n1-standard-4" + DefaultGKERegion = "us-central1" + DefaultGKEZone = "" + regionEnv = "E2E_CLUSTER_REGION" + backupRegionEnv = "E2E_CLUSTER_BACKUP_REGIONS" + defaultGKEVersion = "latest" + + ClusterRunning = "RUNNING" +) + +var ( + DefaultGKEBackupRegions = []string{"us-west1", "us-east1"} + protectedProjects = []string{"knative-tests"} + protectedClusters = []string{"knative-prow"} +) + +// GKEClient implements Client +type GKEClient struct { +} + +// GKERequest contains all requests collected for cluster creation +type GKERequest struct { + // Request holds settings for GKE native operations + gke.Request + + // BackupRegions: fall back regions to try out in case of cluster creation + // failure due to regional issue(s) + BackupRegions []string + + // SkipCreation: skips cluster creation + SkipCreation bool + + // NeedsCleanup: enforce clean up if given this option, used when running + // locally + NeedsCleanup bool +} + +// GKECluster implements ClusterOperations +type GKECluster struct { + Request *GKERequest + // Project might be GKE specific, so put it here + Project string + // NeedsCleanup tells whether the cluster needs to be deleted afterwards + // This probably should be part of task wrapper's logic + NeedsCleanup bool + Cluster *container.Cluster + operations gke.SDKOperations + boskosOps boskos.Operation +} + +// Setup sets up a GKECluster client, takes GEKRequest as parameter and applies +// all defaults if not defined. +func (gs *GKEClient) Setup(r GKERequest) ClusterOperations { + gc := &GKECluster{} + + if r.Project != "" { // use provided project and create cluster + gc.Project = r.Project + gc.NeedsCleanup = true + } + + if r.ClusterName == "" { + var err error + r.ClusterName, err = getResourceName(ClusterResource) + if err != nil { + log.Fatalf("Failed getting cluster name: '%v'", err) + } + } + + if r.MinNodes == 0 { + r.MinNodes = DefaultGKEMinNodes + } + if r.MaxNodes == 0 { + r.MaxNodes = DefaultGKEMaxNodes + // We don't want MaxNodes < MinNodes + if r.MinNodes > r.MaxNodes { + r.MaxNodes = r.MinNodes + } + } + if r.NodeType == "" { + r.NodeType = DefaultGKENodeType + } + // Only use default backup regions if region is not provided + if len(r.BackupRegions) == 0 && r.Region == "" { + r.BackupRegions = DefaultGKEBackupRegions + if common.GetOSEnv(backupRegionEnv) != "" { + r.BackupRegions = strings.Split(common.GetOSEnv(backupRegionEnv), " ") + } + } + if r.Region == "" { + r.Region = DefaultGKERegion + if common.GetOSEnv(regionEnv) != "" { + r.Region = common.GetOSEnv(regionEnv) + } + } + if r.Zone == "" { + r.Zone = DefaultGKEZone + } else { // No backupregions if zone is provided + r.BackupRegions = make([]string, 0) + } + + gc.Request = &r + + client, err := gke.NewSDKClient() + if err != nil { + log.Fatalf("failed to create GKE SDK client: '%v'", err) + } + gc.operations = client + + gc.boskosOps = &boskos.Client{} + + return gc +} + +// Provider returns gke +func (gc *GKECluster) Provider() string { + return "gke" +} + +// Acquire gets existing cluster or create a new one, the creation logic +// contains retries in BackupRegions. Default creating cluster +// in us-central1, and default BackupRegions are us-west1 and us-east1. If +// Region or Zone is provided then there is no retries +func (gc *GKECluster) Acquire() error { + if err := gc.checkEnvironment(); err != nil { + return fmt.Errorf("failed checking project/cluster from environment: '%v'", err) + } + // If gc.Cluster is discovered above, then the cluster exists and it's + // project and name matches with requested, use it + if gc.Cluster != nil { + gc.ensureProtected() + return nil + } + if gc.Request.SkipCreation { + return errors.New("cannot acquire cluster if SkipCreation is set") + } + + // If comes here we are very likely going to create a cluster, unless + // the cluster already exists + + // Cleanup if cluster is created by this client + gc.NeedsCleanup = !common.IsProw() + + // Get project name from boskos if running in Prow, otherwise it should fail + // since we don't know which project to use + if common.IsProw() { + project, err := gc.boskosOps.AcquireGKEProject(nil) + if err != nil { + return fmt.Errorf("failed acquiring boskos project: '%v'", err) + } + gc.Project = project.Name + } + if gc.Project == "" { + return errors.New("GCP project must be set") + } + gc.ensureProtected() + log.Printf("Identified project %s for cluster creation", gc.Project) + + // Make a deep copy of the request struct, since the original request is supposed to be immutable + request := gc.Request.DeepCopy() + // We are going to use request for creating cluster, set its Project + request.Project = gc.Project + + // Combine Region with BackupRegions, these will be the regions used for + // retrying creation logic + regions := []string{request.Region} + for _, br := range gc.Request.BackupRegions { + if br != request.Region { + regions = append(regions, br) + } + } + var cluster *container.Cluster + rb, err := gke.NewCreateClusterRequest(request) + if err != nil { + return fmt.Errorf("failed building the CreateClusterRequest: '%v'", err) + } + for i, region := range regions { + // Restore innocence + err = nil + + clusterName := request.ClusterName + // Use cluster if it already exists and running + existingCluster, _ := gc.operations.GetCluster(gc.Project, region, request.Zone, clusterName) + if existingCluster != nil && existingCluster.Status == ClusterRunning { + gc.Cluster = existingCluster + return nil + } + // Creating cluster + log.Printf("Creating cluster %q in region %q zone %q with:\n%+v", clusterName, region, request.Zone, gc.Request) + err = gc.operations.CreateCluster(gc.Project, region, request.Zone, rb) + if err == nil { + cluster, err = gc.operations.GetCluster(gc.Project, region, request.Zone, rb.Cluster.Name) + } + if err != nil { + errMsg := fmt.Sprintf("Error during cluster creation: '%v'. ", err) + if gc.NeedsCleanup { // Delete half created cluster if it's user created + errMsg = fmt.Sprintf("%sDeleting cluster %q in region %q zone %q in background...\n", errMsg, clusterName, region, request.Zone) + gc.operations.DeleteClusterAsync(gc.Project, region, request.Zone, clusterName) + } + // Retry another region if cluster creation failed. + // TODO(chaodaiG): catch specific errors as we know what the error look like for stockout etc. + if i != len(regions)-1 { + errMsg = fmt.Sprintf("%sRetry another region %q for cluster creation", errMsg, regions[i+1]) + } + log.Printf(errMsg) + } else { + log.Print("Cluster creation completed") + gc.Cluster = cluster + break + } + } + + return err +} + +// Delete takes care of GKE cluster resource cleanup. It only release Boskos resource if running in +// Prow, otherwise deletes the cluster if marked NeedsCleanup +func (gc *GKECluster) Delete() error { + if err := gc.checkEnvironment(); err != nil { + return fmt.Errorf("failed checking project/cluster from environment: '%v'", err) + } + gc.ensureProtected() + // Release Boskos if running in Prow, will let Janitor taking care of + // clusters deleting + if common.IsProw() { + log.Printf("Releasing Boskos resource: '%v'", gc.Project) + return gc.boskosOps.ReleaseGKEProject(nil, gc.Project) + } + + // NeedsCleanup is only true if running locally and cluster created by the + // process + if !gc.NeedsCleanup && !gc.Request.NeedsCleanup { + return nil + } + // Should only get here if running locally and cluster created by this + // client, so at this moment cluster should have been set + if gc.Cluster == nil { + return fmt.Errorf("cluster doesn't exist") + } + log.Printf("Deleting cluster %q in %q", gc.Cluster.Name, gc.Cluster.Location) + region, zone := gke.RegionZoneFromLoc(gc.Cluster.Location) + if err := gc.operations.DeleteCluster(gc.Project, region, zone, gc.Cluster.Name); err != nil { + return fmt.Errorf("failed deleting cluster: '%v'", err) + } + return nil +} + +// ensureProtected ensures not operating on protected project/cluster +func (gc *GKECluster) ensureProtected() { + if gc.Project != "" { + for _, pp := range protectedProjects { + if gc.Project == pp { + log.Fatalf("project %q is protected", gc.Project) + } + } + } + if gc.Cluster != nil { + for _, pc := range protectedClusters { + if gc.Cluster.Name == pc { + log.Fatalf("cluster %q is protected", gc.Cluster.Name) + } + } + } +} + +// checkEnvironment checks environment set for kubeconfig and gcloud, and try to +// identify existing project/cluster if they are not set +// +// checks for existing cluster by looking at kubeconfig, if kubeconfig is set: +// - If it exists in GKE: +// - If Request doesn't contain project/clustername: +// - Use it +// - If Request contains any of project/clustername: +// - If the cluster matches with them: +// - Use it +// If cluster isn't discovered above, try to get project from gcloud +func (gc *GKECluster) checkEnvironment() error { + output, err := common.StandardExec("kubectl", "config", "current-context") + // if kubeconfig is configured, try to use it + if err == nil { + currentContext := strings.TrimSpace(string(output)) + log.Printf("kubeconfig is: %q", currentContext) + if strings.HasPrefix(currentContext, "gke_") { + // output should be in the form of gke_PROJECT_REGION_CLUSTER + parts := strings.Split(currentContext, "_") + if len(parts) != 4 { // fall through with warning + log.Printf("WARNING: ignoring kubectl current-context since it's malformed: %q", currentContext) + } else { + project := parts[1] + location, clusterName := parts[2], parts[3] + region, zone := gke.RegionZoneFromLoc(location) + // Use the cluster only if project and clustername match + if (gc.Request.Project == "" || gc.Request.Project == project) && (gc.Request.ClusterName == "" || gc.Request.ClusterName == clusterName) { + cluster, err := gc.operations.GetCluster(project, region, zone, clusterName) + if err != nil { + return fmt.Errorf("couldn't find cluster %s in %s in %s, does it exist? %v", clusterName, project, location, err) + } + gc.Cluster = cluster + gc.Project = project + } + return nil + } + } + } + // When kubeconfig isn't set, the err isn't nil and output should be empty. + // If output isn't empty then this is unexpected error, should shout out + // directly + if err != nil && len(output) > 0 { + return fmt.Errorf("failed running kubectl config current-context: '%s'", string(output)) + } + + if gc.Project != "" { + return nil + } + + // if gcloud is pointing to a project, use it + output, err = common.StandardExec("gcloud", "config", "get-value", "project") + if err != nil { + return fmt.Errorf("failed getting gcloud project: '%v'", err) + } + if string(output) != "" { + project := strings.Trim(strings.TrimSpace(string(output)), "\n\r") + gc.Project = project + } + return nil +} diff --git a/vendor/knative.dev/pkg/testutils/clustermanager/util.go b/vendor/knative.dev/pkg/testutils/clustermanager/e2e-tests/util.go similarity index 89% rename from vendor/knative.dev/pkg/testutils/clustermanager/util.go rename to vendor/knative.dev/pkg/testutils/clustermanager/e2e-tests/util.go index f905e7b6..9d64aec5 100644 --- a/vendor/knative.dev/pkg/testutils/clustermanager/util.go +++ b/vendor/knative.dev/pkg/testutils/clustermanager/e2e-tests/util.go @@ -19,7 +19,7 @@ package clustermanager import ( "fmt" - "knative.dev/pkg/testutils/common" + "knative.dev/pkg/testutils/clustermanager/e2e-tests/common" ) var ( @@ -51,10 +51,3 @@ func getResourceName(rt ResourceType) (string, error) { } return resName, nil } - -func getClusterLocation(region, zone string) string { - if zone != "" { - region = fmt.Sprintf("%s-%s", region, zone) - } - return region -} diff --git a/vendor/knative.dev/pkg/testutils/clustermanager/gke.go b/vendor/knative.dev/pkg/testutils/clustermanager/gke.go deleted file mode 100644 index db1234eb..00000000 --- a/vendor/knative.dev/pkg/testutils/clustermanager/gke.go +++ /dev/null @@ -1,542 +0,0 @@ -/* -Copyright 2019 The Knative Authors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package clustermanager - -import ( - "errors" - "fmt" - "log" - "strings" - "time" - - container "google.golang.org/api/container/v1beta1" - "knative.dev/pkg/testutils/clustermanager/boskos" - "knative.dev/pkg/testutils/common" - - "golang.org/x/net/context" - "golang.org/x/oauth2/google" -) - -const ( - DefaultGKEMinNodes = 1 - DefaultGKEMaxNodes = 3 - DefaultGKENodeType = "n1-standard-4" - DefaultGKERegion = "us-central1" - DefaultGKEZone = "" - regionEnv = "E2E_CLUSTER_REGION" - backupRegionEnv = "E2E_CLUSTER_BACKUP_REGIONS" - defaultGKEVersion = "latest" -) - -var ( - DefaultGKEBackupRegions = []string{"us-west1", "us-east1"} - protectedProjects = []string{"knative-tests"} - protectedClusters = []string{"knative-prow"} - // These are arbitrary numbers determined based on past experience - creationTimeout = 20 * time.Minute - deletionTimeout = 10 * time.Minute - autoscalingTimeout = 1 * time.Minute -) - -// GKEClient implements Client -type GKEClient struct { -} - -// GKERequest contains all requests collected for cluster creation -type GKERequest struct { - // Project: GKE project, no default. Fall back to get project from kubeconfig - // then gcloud config - Project string - - // ClusterName: custom cluster name to use. Fall back to cluster set by - // kubeconfig, else composed as k[REPO]-cls-e2e-[BUILD_ID] - ClusterName string - - // MinNodes: default to 1 if not provided - MinNodes int64 - - // MaxNodes: default to max(3, MinNodes) if not provided - MaxNodes int64 - - // NodeType: default to n1-standard-4 if not provided - NodeType string - - // Region: default to regional cluster if not provided, and use default backup regions - Region string - - // Zone: default is none, must be provided together with region - Zone string - - // BackupRegions: fall back regions to try out in case of cluster creation - // failure due to regional issue(s) - BackupRegions []string - - // Addons: cluster addons to be added to cluster, such as istio - Addons []string - - // SkipCreation: skips cluster creation - SkipCreation bool - - // NeedsCleanup: enforce clean up if given this option, used when running - // locally - NeedsCleanup bool -} - -// GKECluster implements ClusterOperations -type GKECluster struct { - Request *GKERequest - // Project might be GKE specific, so put it here - Project *string - // NeedsCleanup tells whether the cluster needs to be deleted afterwards - // This probably should be part of task wrapper's logic - NeedsCleanup bool - Cluster *container.Cluster - operations GKESDKOperations - boskosOps boskos.Operation -} - -// GKESDKOperations wraps GKE SDK related functions -type GKESDKOperations interface { - create(string, string, *container.CreateClusterRequest) (*container.Operation, error) - delete(string, string, string) (*container.Operation, error) - get(string, string, string) (*container.Cluster, error) - getOperation(string, string, string) (*container.Operation, error) - setAutoscaling(string, string, string, string, *container.SetNodePoolAutoscalingRequest) (*container.Operation, error) -} - -// GKESDKClient Implement GKESDKOperations -type GKESDKClient struct { - *container.Service -} - -func (gsc *GKESDKClient) create(project, location string, rb *container.CreateClusterRequest) (*container.Operation, error) { - parent := fmt.Sprintf("projects/%s/locations/%s", project, location) - return gsc.Projects.Locations.Clusters.Create(parent, rb).Context(context.Background()).Do() -} - -// delete deletes GKE cluster and waits until completion -func (gsc *GKESDKClient) delete(project, clusterName, location string) (*container.Operation, error) { - parent := fmt.Sprintf("projects/%s/locations/%s/clusters/%s", project, location, clusterName) - return gsc.Projects.Locations.Clusters.Delete(parent).Context(context.Background()).Do() -} - -func (gsc *GKESDKClient) get(project, location, cluster string) (*container.Cluster, error) { - clusterFullPath := fmt.Sprintf("projects/%s/locations/%s/clusters/%s", project, location, cluster) - return gsc.Projects.Locations.Clusters.Get(clusterFullPath).Context(context.Background()).Do() -} - -func (gsc *GKESDKClient) getOperation(project, location, opName string) (*container.Operation, error) { - name := fmt.Sprintf("projects/%s/locations/%s/operations/%s", project, location, opName) - return gsc.Service.Projects.Locations.Operations.Get(name).Do() -} - -// setAutoscaling sets up autoscaling for a nodepool. This function is not -// covered by either `Clusters.Update` or `NodePools.Update`, so can not really -// make it as generic as the others -func (gsc *GKESDKClient) setAutoscaling(project, clusterName, location, nodepoolName string, - rb *container.SetNodePoolAutoscalingRequest) (*container.Operation, error) { - parent := fmt.Sprintf("projects/%s/locations/%s/clusters/%s/nodePools/%s", project, location, clusterName, nodepoolName) - return gsc.Service.Projects.Locations.Clusters.NodePools.SetAutoscaling(parent, rb).Do() -} - -// Setup sets up a GKECluster client, takes GEKRequest as parameter and applies -// all defaults if not defined. -func (gs *GKEClient) Setup(r GKERequest) ClusterOperations { - gc := &GKECluster{} - - if r.Project != "" { // use provided project and create cluster - gc.Project = &r.Project - gc.NeedsCleanup = true - } - - if r.MinNodes == 0 { - r.MinNodes = DefaultGKEMinNodes - } - if r.MaxNodes == 0 { - r.MaxNodes = DefaultGKEMaxNodes - // We don't want MaxNodes < MinNodes - if r.MinNodes > r.MaxNodes { - r.MaxNodes = r.MinNodes - } - } - if r.NodeType == "" { - r.NodeType = DefaultGKENodeType - } - // Only use default backup regions if region is not provided - if len(r.BackupRegions) == 0 && r.Region == "" { - r.BackupRegions = DefaultGKEBackupRegions - if common.GetOSEnv(backupRegionEnv) != "" { - r.BackupRegions = strings.Split(common.GetOSEnv(backupRegionEnv), " ") - } - } - if r.Region == "" { - r.Region = DefaultGKERegion - if common.GetOSEnv(regionEnv) != "" { - r.Region = common.GetOSEnv(regionEnv) - } - } - if r.Zone == "" { - r.Zone = DefaultGKEZone - } else { // No backupregions if zone is provided - r.BackupRegions = make([]string, 0) - } - - gc.Request = &r - - ctx := context.Background() - c, err := google.DefaultClient(ctx, container.CloudPlatformScope) - if err != nil { - log.Fatalf("failed create google client: '%v'", err) - } - - containerService, err := container.New(c) - if err != nil { - log.Fatalf("failed create container service: '%v'", err) - } - gc.operations = &GKESDKClient{containerService} - - gc.boskosOps = &boskos.Client{} - - return gc -} - -// initialize checks environment for cluster and projects to decide whether using -// existing cluster/project or creating new ones. -func (gc *GKECluster) initialize() error { - // Try obtain project name via `kubectl`, `gcloud` - if gc.Project == nil { - if err := gc.checkEnvironment(); err != nil { - return fmt.Errorf("failed checking existing cluster: '%v'", err) - } else if gc.Cluster != nil { // Return if Cluster was already set by kubeconfig - // If clustername provided and kubeconfig set, ignore kubeconfig - if gc.Request != nil && gc.Request.ClusterName != "" && gc.Cluster.Name != gc.Request.ClusterName { - gc.Cluster = nil - } - if gc.Cluster != nil { - return nil - } - } - } - // Get project name from boskos if running in Prow - if gc.Project == nil && common.IsProw() { - project, err := gc.boskosOps.AcquireGKEProject(nil) - if err != nil { - return fmt.Errorf("failed acquire boskos project: '%v'", err) - } - gc.Project = &project.Name - } - if gc.Project == nil || *gc.Project == "" { - return errors.New("gcp project must be set") - } - if !common.IsProw() && gc.Cluster == nil { - gc.NeedsCleanup = true - } - log.Printf("Using project %q for running test", *gc.Project) - return nil -} - -// Provider returns gke -func (gc *GKECluster) Provider() string { - return "gke" -} - -// Acquire gets existing cluster or create a new one, the creation logic -// contains retries in BackupRegions. Default creating cluster -// in us-central1, and default BackupRegions are us-west1 and us-east1. If -// Region or Zone is provided then there is no retries -func (gc *GKECluster) Acquire() error { - if err := gc.initialize(); err != nil { - return fmt.Errorf("failed initialing with environment: '%v'", err) - } - gc.ensureProtected() - clusterName := gc.Request.ClusterName - var err error - // Check if using existing cluster - if gc.Cluster != nil { - return nil - } - if gc.Request.SkipCreation { - log.Println("Skipping cluster creation as SkipCreation is set") - return nil - } - // Perform GKE specific cluster creation logics - if gc.Request.ClusterName == "" { - clusterName, err = getResourceName(ClusterResource) - if err != nil { - return fmt.Errorf("failed getting cluster name: '%v'", err) - } - } - - regions := []string{gc.Request.Region} - for _, br := range gc.Request.BackupRegions { - exist := false - for _, region := range regions { - if br == region { - exist = true - } - } - if !exist { - regions = append(regions, br) - } - } - var cluster *container.Cluster - var op *container.Operation - for i, region := range regions { - // Restore innocence - err = nil - rb := &container.CreateClusterRequest{ - Cluster: &container.Cluster{ - Name: clusterName, - // The default cluster version is not latest, has to explicitly - // set it as "latest" - InitialClusterVersion: defaultGKEVersion, - // Installing addons after cluster creation takes at least 5 - // minutes, so install addons as part of cluster creation, which - // doesn't seem to add much time on top of cluster creation - AddonsConfig: gc.getAddonsConfig(), - // Equivalent to --enable-basic-auth, so that user:pass can be - // later on retrieved for setting up cluster roles. Use the - // default username from gcloud command, the password will be - // automatically generated by GKE SDK - MasterAuth: &container.MasterAuth{Username: "admin"}, - InitialNodeCount: gc.Request.MinNodes, - NodeConfig: &container.NodeConfig{ - MachineType: gc.Request.NodeType, - }, - }, - ProjectId: *gc.Project, - } - - clusterLoc := getClusterLocation(region, gc.Request.Zone) - - // Deleting cluster if it already exists - existingCluster, _ := gc.operations.get(*gc.Project, clusterLoc, clusterName) - if existingCluster != nil { - log.Printf("Cluster %q already exists in %q. Deleting...", clusterName, clusterLoc) - op, err = gc.operations.delete(*gc.Project, clusterName, clusterLoc) - if err == nil { - err = gc.wait(clusterLoc, op.Name, deletionTimeout) - } - } - // Creating cluster only if previous step succeeded - if err == nil { - log.Printf("Creating cluster %q in %q with:\n%+v", clusterName, clusterLoc, gc.Request) - op, err = gc.operations.create(*gc.Project, clusterLoc, rb) - if err == nil { - err = gc.wait(clusterLoc, op.Name, creationTimeout) - } - if err == nil { // Enable autoscaling and set limits - arb := &container.SetNodePoolAutoscalingRequest{ - Autoscaling: &container.NodePoolAutoscaling{ - Enabled: true, - MinNodeCount: gc.Request.MinNodes, - MaxNodeCount: gc.Request.MaxNodes, - }, - } - - op, err = gc.operations.setAutoscaling(*gc.Project, clusterName, clusterLoc, "default-pool", arb) - if err == nil { - err = gc.wait(clusterLoc, op.Name, autoscalingTimeout) - } - } - if err == nil { // Get cluster at last - cluster, err = gc.operations.get(*gc.Project, clusterLoc, rb.Cluster.Name) - } - } - if err != nil { - errMsg := fmt.Sprintf("Error during cluster creation: '%v'. ", err) - if gc.NeedsCleanup { // Delete half created cluster if it's user created - errMsg = fmt.Sprintf("%sDeleting cluster %q in %q in background...\n", errMsg, clusterName, clusterLoc) - go gc.operations.delete(*gc.Project, clusterName, clusterLoc) - } - // Retry another region if cluster creation failed. - // TODO(chaodaiG): catch specific errors as we know what the error look like for stockout etc. - if i != len(regions)-1 { - errMsg = fmt.Sprintf("%sRetry another region %q for cluster creation", errMsg, regions[i+1]) - } - log.Printf(errMsg) - } else { - log.Print("Cluster creation completed") - gc.Cluster = cluster - break - } - } - - return err -} - -// Delete takes care of GKE cluster resource cleanup. It only release Boskos resource if running in -// Prow, otherwise deletes the cluster if marked NeedsCleanup -func (gc *GKECluster) Delete() error { - if err := gc.initialize(); err != nil { - return fmt.Errorf("failed initialing with environment: '%v'", err) - } - gc.ensureProtected() - // Release Boskos if running in Prow, will let Janitor taking care of - // clusters deleting - if common.IsProw() { - log.Printf("Releasing Boskos resource: '%v'", *gc.Project) - return gc.boskosOps.ReleaseGKEProject(nil, *gc.Project) - } - - // NeedsCleanup is only true if running locally and cluster created by the - // process - if !gc.NeedsCleanup && !gc.Request.NeedsCleanup { - return nil - } - // Should only get here if running locally and cluster created by this - // client, so at this moment cluster should have been set - if gc.Cluster == nil { - return fmt.Errorf("cluster doesn't exist") - } - - log.Printf("Deleting cluster %q in %q", gc.Cluster.Name, gc.Cluster.Location) - op, err := gc.operations.delete(*gc.Project, gc.Cluster.Name, gc.Cluster.Location) - if err == nil { - err = gc.wait(gc.Cluster.Location, op.Name, deletionTimeout) - } - if err != nil { - return fmt.Errorf("failed deleting cluster: '%v'", err) - } - return nil -} - -// getAddonsConfig gets AddonsConfig from Request, contains the logic of -// converting string argument to typed AddonsConfig, for example `IstioConfig`. -// Currently supports istio -func (gc *GKECluster) getAddonsConfig() *container.AddonsConfig { - const ( - // Define all supported addons here - istio = "istio" - ) - ac := &container.AddonsConfig{} - for _, name := range gc.Request.Addons { - switch strings.ToLower(name) { - case istio: - ac.IstioConfig = &container.IstioConfig{Disabled: false} - default: - panic(fmt.Sprintf("addon type %q not supported. Has to be one of: %q", name, istio)) - } - } - - return ac -} - -// wait depends on unique opName(operation ID created by cloud), and waits until -// it's done -func (gc *GKECluster) wait(location, opName string, wait time.Duration) error { - const ( - pendingStatus = "PENDING" - runningStatus = "RUNNING" - doneStatus = "DONE" - ) - var op *container.Operation - var err error - - timeout := time.After(wait) - tick := time.Tick(500 * time.Millisecond) - for { - select { - // Got a timeout! fail with a timeout error - case <-timeout: - return errors.New("timed out waiting") - case <-tick: - // Retry 3 times in case of weird network error, or rate limiting - for r, w := 0, 50*time.Microsecond; r < 3; r, w = r+1, w*2 { - op, err = gc.operations.getOperation(*gc.Project, location, opName) - if err == nil { - if op.Status == doneStatus { - return nil - } else if op.Status == pendingStatus || op.Status == runningStatus { - // Valid operation, no need to retry - break - } else { - // Have seen intermittent error state and fixed itself, - // let it retry to avoid too much flakiness - err = fmt.Errorf("unexpected operation status: %q", op.Status) - } - } - time.Sleep(w) - } - // If err still persist after retries, exit - if err != nil { - return err - } - } - } -} - -// ensureProtected ensures not operating on protected project/cluster -func (gc *GKECluster) ensureProtected() { - if gc.Project != nil { - for _, pp := range protectedProjects { - if *gc.Project == pp { - log.Fatalf("project %q is protected", *gc.Project) - } - } - } - if gc.Cluster != nil { - for _, pc := range protectedClusters { - if gc.Cluster.Name == pc { - log.Fatalf("cluster %q is protected", gc.Cluster.Name) - } - } - } -} - -// checks for existing cluster by looking at kubeconfig, -// and sets up gc.Project and gc.Cluster properly, otherwise fail it. -// if project can be derived from gcloud, sets it up as well -func (gc *GKECluster) checkEnvironment() error { - var err error - // if kubeconfig is configured, use it - output, err := common.StandardExec("kubectl", "config", "current-context") - if err == nil { - currentContext := strings.TrimSpace(string(output)) - if strings.HasPrefix(currentContext, "gke_") { - // output should be in the form of gke_PROJECT_REGION_CLUSTER - parts := strings.Split(currentContext, "_") - if len(parts) != 4 { // fall through with warning - log.Printf("WARNING: ignoring kubectl current-context since it's malformed: '%s'", currentContext) - } else { - log.Printf("kubeconfig isn't empty, uses this cluster for running tests: %s", currentContext) - gc.Project = &parts[1] - gc.Cluster, err = gc.operations.get(*gc.Project, parts[2], parts[3]) - if err != nil { - return fmt.Errorf("couldn't find cluster %s in %s in %s, does it exist? %v", parts[3], parts[1], parts[2], err) - } - return nil - } - } - } - if err != nil && len(output) > 0 { - // this is unexpected error, should shout out directly - return fmt.Errorf("failed running kubectl config current-context: '%s'", string(output)) - } - - // if gcloud is pointing to a project, use it - output, err = common.StandardExec("gcloud", "config", "get-value", "project") - if err != nil { - return fmt.Errorf("failed getting gcloud project: '%v'", err) - } - if string(output) != "" { - project := strings.Trim(strings.TrimSpace(string(output)), "\n\r") - gc.Project = &project - } - - return nil -} diff --git a/vendor/knative.dev/pkg/testutils/clustermanager/perf-tests/main.go b/vendor/knative.dev/pkg/testutils/clustermanager/perf-tests/main.go new file mode 100644 index 00000000..7005a18e --- /dev/null +++ b/vendor/knative.dev/pkg/testutils/clustermanager/perf-tests/main.go @@ -0,0 +1,65 @@ +/* +Copyright 2019 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "flag" + "log" + + testPkg "knative.dev/pkg/testutils/clustermanager/perf-tests/pkg" +) + +// flags supported by this tool +var ( + isRecreate bool + isReconcile bool + gcpProjectName string + repoName string + benchmarkRootFolder string +) + +func main() { + flag.StringVar(&gcpProjectName, "gcp-project", "", "name of the GCP project for cluster operations") + flag.StringVar(&repoName, "repository", "", "name of the repository") + flag.StringVar(&benchmarkRootFolder, "benchmark-root", "", "root folder of the benchmarks") + flag.BoolVar(&isRecreate, "recreate", false, "is recreate operation or not") + flag.BoolVar(&isReconcile, "reconcile", false, "is reconcile operation or not") + flag.Parse() + + if isRecreate && isReconcile { + log.Fatal("Only one operation can be specified, either recreate or reconcile") + } + + client, err := testPkg.NewClient() + if err != nil { + log.Fatalf("Failed setting up GKE client, cannot proceed: %v", err) + } + switch { + case isRecreate: + if err := client.RecreateClusters(gcpProjectName, repoName, benchmarkRootFolder); err != nil { + log.Fatalf("Failed recreating clusters for repo %q: %v", repoName, err) + } + log.Printf("Done with recreating clusters for repo %q", repoName) + case isReconcile: + if err := client.ReconcileClusters(gcpProjectName, repoName, benchmarkRootFolder); err != nil { + log.Fatalf("Failed reconciling clusters for repo %q: %v", repoName, err) + } + log.Printf("Done with reconciling clusters for repo %q", repoName) + default: + log.Fatal("One operation must be specified, either recreate or reconcile") + } +} diff --git a/vendor/knative.dev/pkg/testutils/clustermanager/perf-tests/pkg/benchmark.go b/vendor/knative.dev/pkg/testutils/clustermanager/perf-tests/pkg/benchmark.go new file mode 100644 index 00000000..96e9a7c7 --- /dev/null +++ b/vendor/knative.dev/pkg/testutils/clustermanager/perf-tests/pkg/benchmark.go @@ -0,0 +1,157 @@ +/* +Copyright 2019 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package pkg + +import ( + "fmt" + "io/ioutil" + "log" + "os" + "path/filepath" + "strings" + + yaml "gopkg.in/yaml.v2" +) + +const ( + // clusterConfigFile is the config file needs to be put under the benchmark folder if we want to config + // the cluster that runs the benchmark, it must follow the scheme defined as GKECluster here. + clusterConfigFile = "cluster.yaml" + + // These default settings will be used for configuring the cluster, if not specified in cluster.yaml. + defaultLocation = "us-central1" + defaultNodeCount = 1 + defaultNodeType = "n1-standard-4" + defaultAddons = "" +) + +// backupLocations are used in retrying cluster creation, if stockout happens in one location. +// TODO(chizhg): it's currently not used, use it in the cluster creation retry logic. +var backupLocations = []string{"us-west1", "us-west2", "us-east1"} + +// GKECluster saves the config information for the GKE cluster +type GKECluster struct { + Config ClusterConfig `yaml:"GKECluster,omitempty"` +} + +// ClusterConfig is config for the cluster +type ClusterConfig struct { + Location string `yaml:"location,omitempty"` + NodeCount int64 `yaml:"nodeCount,omitempty"` + NodeType string `yaml:"nodeType,omitempty"` + Addons string `yaml:"addons,omitempty"` +} + +// benchmarkNames returns names of the benchmarks. +// +// We put all benchmarks under the benchmarkRoot folder, one subfolder represents one benchmark, +// here we returns all subfolder names of the root folder. +func benchmarkNames(benchmarkRoot string) ([]string, error) { + names := make([]string, 0) + dirs, err := ioutil.ReadDir(benchmarkRoot) + if err != nil { + return names, fmt.Errorf("failed to list all benchmarks under %q: %v", benchmarkRoot, err) + } + + for _, dir := range dirs { + names = append(names, dir.Name()) + } + return names, nil +} + +// benchmarkClusters returns the cluster configs for all benchmarks. +func benchmarkClusters(repo, benchmarkRoot string) (map[string]ClusterConfig, error) { + // clusters is a map of cluster configs + // key is the cluster name, value is the cluster config + clusters := make(map[string]ClusterConfig) + benchmarkNames, err := benchmarkNames(benchmarkRoot) + if err != nil { + return clusters, err + } + + for _, benchmarkName := range benchmarkNames { + clusterConfig := clusterConfigForBenchmark(benchmarkName, benchmarkRoot) + clusterName := clusterNameForBenchmark(benchmarkName, repo) + clusters[clusterName] = clusterConfig + } + + return clusters, nil +} + +// clusterConfigForBenchmark returns the cluster config for the given benchmark. +// +// Under each benchmark folder, we can put a cluster.yaml file that follows the scheme we define +// in ClusterConfig struct, in which we specify configuration of the cluster that we use to run the benchmark. +// If there is no such config file, or the config file is malformed, default config will be used. +func clusterConfigForBenchmark(benchmarkName, benchmarkRoot string) ClusterConfig { + gkeCluster := GKECluster{ + Config: ClusterConfig{ + Location: defaultLocation, + NodeCount: defaultNodeCount, + NodeType: defaultNodeType, + Addons: defaultAddons, + }, + } + + configFile := filepath.Join(benchmarkRoot, benchmarkName, clusterConfigFile) + if fileExists(configFile) { + contents, err := ioutil.ReadFile(configFile) + if err == nil { + if err := yaml.Unmarshal(contents, &gkeCluster); err != nil { + log.Printf("WARNING: cannot parse the config file %q, default config will be used", configFile) + } + } else { + log.Printf("WARNING: cannot read the config file %q, default config will be used", configFile) + } + } + + return gkeCluster.Config +} + +// clusterNameForBenchmark prepends repo name to the benchmark name, and use it as the cluster name. +func clusterNameForBenchmark(benchmarkName, repo string) string { + return repoPrefix(repo) + benchmarkName +} + +// benchmarkNameForCluster removes repo name prefix from the cluster name, to get the real benchmark name. +// If the cluster does not belong to the given repo, return an empty string. +func benchmarkNameForCluster(clusterName, repo string) string { + if !clusterBelongsToRepo(clusterName, repo) { + return "" + } + return strings.TrimPrefix(clusterName, repoPrefix(repo)) +} + +// clusterBelongsToRepo determines if the cluster belongs to the repo, by checking if it has the repo prefix. +func clusterBelongsToRepo(clusterName, repo string) bool { + return strings.HasPrefix(clusterName, repoPrefix(repo)) +} + +// repoPrefix returns the prefix we want to add to the benchmark name and use as the cluster name. +// This is needed to distinguish between different repos if they are using a same GCP project. +func repoPrefix(repo string) string { + return repo + "--" +} + +// fileExists returns if the file exists or not +func fileExists(fileName string) bool { + info, err := os.Stat(fileName) + if os.IsNotExist(err) { + return false + } + return !info.IsDir() +} diff --git a/vendor/knative.dev/pkg/testutils/clustermanager/perf-tests/pkg/cluster.go b/vendor/knative.dev/pkg/testutils/clustermanager/perf-tests/pkg/cluster.go new file mode 100644 index 00000000..fff45f53 --- /dev/null +++ b/vendor/knative.dev/pkg/testutils/clustermanager/perf-tests/pkg/cluster.go @@ -0,0 +1,264 @@ +/* +Copyright 2019 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package pkg + +import ( + "fmt" + "log" + "strings" + "sync" + + "knative.dev/pkg/test/gke" + "knative.dev/pkg/test/helpers" + + container "google.golang.org/api/container/v1beta1" +) + +const ( + // the maximum retry times if there is an error in cluster operation + retryTimes = 3 + + // known cluster status + // TODO(chizhg): move these status constants to gke package + statusProvisioning = "PROVISIONING" + statusRunning = "RUNNING" + statusStopping = "STOPPING" +) + +type gkeClient struct { + ops gke.SDKOperations +} + +// NewClient will create a new gkeClient. +func NewClient() (*gkeClient, error) { + operations, err := gke.NewSDKClient() + if err != nil { + return nil, fmt.Errorf("failed to set up GKE client: %v", err) + } + + client := &gkeClient{ + ops: operations, + } + return client, nil +} + +// RecreateClusters will delete and recreate the existing clusters. +func (gc *gkeClient) RecreateClusters(gcpProject, repo, benchmarkRoot string) error { + handleExistingCluster := func(cluster container.Cluster, configExists bool, config ClusterConfig) error { + // always delete the cluster, even if the cluster config is unchanged + return gc.handleExistingClusterHelper(gcpProject, cluster, configExists, config, false) + } + handleNewClusterConfig := func(clusterName string, clusterConfig ClusterConfig) error { + // for now, do nothing to the new cluster config + return nil + } + return gc.processClusters(gcpProject, repo, benchmarkRoot, handleExistingCluster, handleNewClusterConfig) +} + +// ReconcileClusters will reconcile all clusters to make them consistent with the benchmarks' cluster configs. +// +// There can be 4 scenarios: +// 1. If the benchmark's cluster config is unchanged, do nothing +// 2. If the benchmark's config is changed, delete the old cluster and create a new one with the new config +// 3. If the benchmark is renamed, delete the old cluster and create a new one with the new name +// 4. If the benchmark is deleted, delete the corresponding cluster +func (gc *gkeClient) ReconcileClusters(gcpProject, repo, benchmarkRoot string) error { + handleExistingCluster := func(cluster container.Cluster, configExists bool, config ClusterConfig) error { + // retain the cluster, if the cluster config is unchanged + return gc.handleExistingClusterHelper(gcpProject, cluster, configExists, config, true) + } + handleNewClusterConfig := func(clusterName string, clusterConfig ClusterConfig) error { + // create a new cluster with the new cluster config + return gc.createClusterWithRetries(gcpProject, clusterName, clusterConfig) + } + return gc.processClusters(gcpProject, repo, benchmarkRoot, handleExistingCluster, handleNewClusterConfig) +} + +// processClusters will process existing clusters and configs for new clusters, +// with the corresponding functions provided by callers. +func (gc *gkeClient) processClusters( + gcpProject, repo, benchmarkRoot string, + handleExistingCluster func(cluster container.Cluster, configExists bool, config ClusterConfig) error, + handleNewClusterConfig func(name string, config ClusterConfig) error, +) error { + curtClusters, err := gc.listClustersForRepo(gcpProject, repo) + if err != nil { + return fmt.Errorf("failed getting clusters for the repo %q: %v", repo, err) + } + clusterConfigs, err := benchmarkClusters(repo, benchmarkRoot) + if err != nil { + return fmt.Errorf("failed getting cluster configs for benchmarks in repo %q: %v", repo, err) + } + + errCh := make(chan error, len(curtClusters)+len(clusterConfigs)) + wg := sync.WaitGroup{} + // handle all existing clusters + for i := range curtClusters { + wg.Add(1) + cluster := curtClusters[i] + config, configExists := clusterConfigs[cluster.Name] + go func() { + defer wg.Done() + if err := handleExistingCluster(cluster, configExists, config); err != nil { + errCh <- fmt.Errorf("failed handling cluster %v: %v", cluster, err) + } + }() + // remove the cluster from clusterConfigs as it's already been handled + delete(clusterConfigs, cluster.Name) + } + + // handle all other cluster configs + for name, config := range clusterConfigs { + wg.Add(1) + // recreate them to avoid the issue with iterations of multiple Go routines + name, config := name, config + go func() { + defer wg.Done() + if err := handleNewClusterConfig(name, config); err != nil { + errCh <- fmt.Errorf("failed handling new cluster config %v: %v", config, err) + } + }() + } + + wg.Wait() + close(errCh) + + errs := make([]error, 0) + for err := range errCh { + if err != nil { + errs = append(errs, err) + } + } + + return helpers.CombineErrors(errs) +} + +// handleExistingClusterHelper is a helper function for handling an existing cluster. +func (gc *gkeClient) handleExistingClusterHelper( + gcpProject string, + cluster container.Cluster, configExists bool, config ClusterConfig, + retainIfUnchanged bool, +) error { + // if the cluster is currently being created or deleted, return directly as that job will handle it properly + if cluster.Status == statusProvisioning || cluster.Status == statusStopping { + log.Printf("Cluster %q is being handled by another job, skip it", cluster.Name) + return nil + } + + curtNodeCount := cluster.CurrentNodeCount + // if it's a regional cluster, the nodes will be in 3 zones. The CurrentNodeCount we get here is + // the total node count, so we'll need to divide with 3 to get the actual regional node count + if _, zone := gke.RegionZoneFromLoc(cluster.Location); zone == "" { + curtNodeCount /= 3 + } + // if retainIfUnchanged is set to true, and the cluster config does not change, do nothing + // TODO(chizhg): also check the addons config + if configExists && retainIfUnchanged && + curtNodeCount == config.NodeCount && cluster.Location == config.Location { + log.Printf("Cluster config is unchanged for %q, skip it", cluster.Name) + return nil + } + + if err := gc.deleteClusterWithRetries(gcpProject, cluster); err != nil { + return fmt.Errorf("failed deleting cluster %q in %q: %v", cluster.Name, cluster.Location, err) + } + if configExists { + return gc.createClusterWithRetries(gcpProject, cluster.Name, config) + } + return nil +} + +// listClustersForRepo will list all the clusters under the gcpProject that belong to the given repo. +func (gc *gkeClient) listClustersForRepo(gcpProject, repo string) ([]container.Cluster, error) { + allClusters, err := gc.ops.ListClustersInProject(gcpProject) + if err != nil { + return nil, fmt.Errorf("failed listing clusters in project %q: %v", gcpProject, err) + } + + clusters := make([]container.Cluster, 0) + for _, cluster := range allClusters { + if clusterBelongsToRepo(cluster.Name, repo) { + clusters = append(clusters, *cluster) + } + } + return clusters, nil +} + +// deleteClusterWithRetries will delete the given cluster, +// and retry for a maximum of retryTimes if there is an error. +// TODO(chizhg): maybe move it to clustermanager library. +func (gc *gkeClient) deleteClusterWithRetries(gcpProject string, cluster container.Cluster) error { + log.Printf("Deleting cluster %q under project %q", cluster.Name, gcpProject) + region, zone := gke.RegionZoneFromLoc(cluster.Location) + var err error + for i := 0; i < retryTimes; i++ { + if err = gc.ops.DeleteCluster(gcpProject, region, zone, cluster.Name); err == nil { + break + } + } + if err != nil { + return fmt.Errorf( + "failed deleting cluster %q in %q after retrying %d times: %v", + cluster.Name, cluster.Location, retryTimes, err) + } + + return nil +} + +// createClusterWithRetries will create a new cluster with the given config, +// and retry for a maximum of retryTimes if there is an error. +// TODO(chizhg): maybe move it to clustermanager library. +func (gc *gkeClient) createClusterWithRetries(gcpProject, name string, config ClusterConfig) error { + log.Printf("Creating cluster %q under project %q with config %v", name, gcpProject, config) + var addons []string + if strings.TrimSpace(config.Addons) != "" { + addons = strings.Split(config.Addons, ",") + } + req := &gke.Request{ + ClusterName: name, + MinNodes: config.NodeCount, + MaxNodes: config.NodeCount, + NodeType: config.NodeType, + Addons: addons, + } + creq, err := gke.NewCreateClusterRequest(req) + if err != nil { + return fmt.Errorf("cannot create cluster with request %v: %v", req, err) + } + + region, zone := gke.RegionZoneFromLoc(config.Location) + for i := 0; i < retryTimes; i++ { + // TODO(chizhg): retry with different requests, based on the error type + if err = gc.ops.CreateCluster(gcpProject, region, zone, creq); err != nil { + // If the cluster is actually created in the end, recreating it with the same name will fail again for sure, + // so we need to delete the broken cluster before retry. + // It is a best-effort delete, and won't throw any errors if the deletion fails. + if cluster, _ := gc.ops.GetCluster(gcpProject, region, zone, name); cluster != nil { + gc.deleteClusterWithRetries(gcpProject, *cluster) + } + } else { + break + } + } + if err != nil { + return fmt.Errorf( + "failed creating cluster %q in %q after retrying %d times: %v", + name, config.Location, retryTimes, err) + } + + return nil +} diff --git a/vendor/knative.dev/pkg/testutils/clustermanager/perf-tests/pkg/testdir/test-benchmark1/cluster.yaml b/vendor/knative.dev/pkg/testutils/clustermanager/perf-tests/pkg/testdir/test-benchmark1/cluster.yaml new file mode 100644 index 00000000..4e1b5f1f --- /dev/null +++ b/vendor/knative.dev/pkg/testutils/clustermanager/perf-tests/pkg/testdir/test-benchmark1/cluster.yaml @@ -0,0 +1,21 @@ +# Copyright 2019 The Knative Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is an example file for a correct cluster config of a benchmark. + +GKECluster: + location: "us-west1" + nodeCount: 4 + nodeType: "n1-standard-8" + addons: "istio" diff --git a/vendor/knative.dev/pkg/testutils/clustermanager/perf-tests/pkg/testdir/test-benchmark2/cluster.yaml b/vendor/knative.dev/pkg/testutils/clustermanager/perf-tests/pkg/testdir/test-benchmark2/cluster.yaml new file mode 100644 index 00000000..7bde4557 --- /dev/null +++ b/vendor/knative.dev/pkg/testutils/clustermanager/perf-tests/pkg/testdir/test-benchmark2/cluster.yaml @@ -0,0 +1,17 @@ +# Copyright 2019 The Knative Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is an example file for a corrupted cluster config of a benchmark. + +anything but not a yaml diff --git a/vendor/knative.dev/pkg/testutils/clustermanager/perf-tests/pkg/testdir/test-benchmark3/cluster.yaml b/vendor/knative.dev/pkg/testutils/clustermanager/perf-tests/pkg/testdir/test-benchmark3/cluster.yaml new file mode 100644 index 00000000..efca4cfd --- /dev/null +++ b/vendor/knative.dev/pkg/testutils/clustermanager/perf-tests/pkg/testdir/test-benchmark3/cluster.yaml @@ -0,0 +1,19 @@ +# Copyright 2019 The Knative Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is an example file for a partial cluster config of a benchmark. + +GKECluster: + nodeCount: 1 + addons: "istio" diff --git a/vendor/knative.dev/pkg/testutils/clustermanager/perf-tests/pkg/testdir/test-benchmark4/noop.yaml b/vendor/knative.dev/pkg/testutils/clustermanager/perf-tests/pkg/testdir/test-benchmark4/noop.yaml new file mode 100644 index 00000000..bab2df31 --- /dev/null +++ b/vendor/knative.dev/pkg/testutils/clustermanager/perf-tests/pkg/testdir/test-benchmark4/noop.yaml @@ -0,0 +1,16 @@ +# Copyright 2019 The Knative Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is added for testing the scenario when cluster.yaml does not exist in the benchmark folder. +# It's necesssary since empty directory will be ignored when we push it to Github. diff --git a/vendor/knative.dev/pkg/testutils/clustermanager/prow-cluster-operation/README.md b/vendor/knative.dev/pkg/testutils/clustermanager/prow-cluster-operation/README.md new file mode 100644 index 00000000..ac587c0a --- /dev/null +++ b/vendor/knative.dev/pkg/testutils/clustermanager/prow-cluster-operation/README.md @@ -0,0 +1,53 @@ +## prow-cluster-operation + +prow-cluster-operation is a tool for creating, deleting, getting a GKE +cluster + +## Prerequisite + +- `GOOGLE_APPLICATION_CREDENTIALS` set + +## Usage + +This tool can be invoked from command line with following parameters: + +- `--min-nodes`: minumum number of nodes, default 1 +- `--max-nodes`: maximum number of nodes, default 3 +- `--node-type`: GCE node type, default "n1-standard-4" +- `--region`: GKE region, default "us-central1" +- `--zone`: GKE zone, default empty +- `--project`: GCP project, default empty +- `--name`: cluster name, default empty +- `--backup-regions`: backup regions to be used if cluster creation in primary + region failed, comma separated list, default "us-west1,us-east1" +- `--addons`: GKE addons, comma separated list, default empty + +## Flow + +### Create + +1. Acquiring cluster if kubeconfig already points to it +1. Get GCP project name if not provided as a parameter: + - [In Prow] Acquire from Boskos + - [Not in Prow] Read from gcloud config + + Failed obtaining project name will fail the tool +1. Get default cluster name if not provided as a parameter +1. Delete cluster if cluster with same name and location already exists in GKE +1. Create cluster +1. Write cluster metadata to `${ARTIFACT}/metadata.json` + +### Delete + +1. Acquiring cluster if kubeconfig already points to it +1. If cluster name is defined then getting cluster by its name +1. If no cluster is found from previous step then it fails +1. Delete: + - [In Prow] Release Boskos project + - [Not in Prow] Delete cluster + +### Get + +1. Acquiring cluster if kubeconfig already points to it +1. If cluster name is defined then getting cluster by its name +1. If no cluster is found from previous step then it fails diff --git a/vendor/knative.dev/pkg/testutils/clustermanager/prow-cluster-operation/actions/create.go b/vendor/knative.dev/pkg/testutils/clustermanager/prow-cluster-operation/actions/create.go new file mode 100644 index 00000000..207a1908 --- /dev/null +++ b/vendor/knative.dev/pkg/testutils/clustermanager/prow-cluster-operation/actions/create.go @@ -0,0 +1,108 @@ +/* +Copyright 2019 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package actions + +import ( + "log" + "strconv" + + container "google.golang.org/api/container/v1beta1" + "knative.dev/pkg/test/gke" + clm "knative.dev/pkg/testutils/clustermanager/e2e-tests" + "knative.dev/pkg/testutils/clustermanager/e2e-tests/common" + "knative.dev/pkg/testutils/clustermanager/prow-cluster-operation/options" + "knative.dev/pkg/testutils/metahelper/client" +) + +const ( + // Keys to be written into metadata.json + e2eRegionKey = "E2E:Region" + e2eZoneKey = "E2E:Zone" + clusterNameKey = "E2E:Machine" + clusterVersionKey = "E2E:Version" + minNodesKey = "E2E:MinNodes" + maxNodesKey = "E2E:MaxNodes" + projectKey = "E2E:Project" +) + +func writeMetaData(cluster *container.Cluster, project string) { + // Set up metadata client for saving metadata + c, err := client.NewClient("") + if err != nil { + log.Fatal(err) + } + log.Printf("Writing metadata to: %q", c.Path) + // Get minNodes and maxNodes counts from default-pool, this is + // usually the case in tests in Prow + var minNodes, maxNodes string + for _, np := range cluster.NodePools { + if np.Name == "default-pool" { + minNodes = strconv.FormatInt(np.InitialNodeCount, 10) + // maxNodes is equal to minNodes if autoscaling isn't on + maxNodes = minNodes + if np.Autoscaling != nil { + minNodes = strconv.FormatInt(np.Autoscaling.MinNodeCount, 10) + maxNodes = strconv.FormatInt(np.Autoscaling.MaxNodeCount, 10) + } else { + log.Printf("DEBUG: nodepool is default-pool but autoscaling is not on: '%+v'", np) + } + break + } + } + + e2eRegion, e2eZone := gke.RegionZoneFromLoc(cluster.Location) + for key, val := range map[string]string{ + e2eRegionKey: e2eRegion, + e2eZoneKey: e2eZone, + clusterNameKey: cluster.Name, + clusterVersionKey: cluster.InitialClusterVersion, + minNodesKey: minNodes, + maxNodesKey: maxNodes, + projectKey: project, + } { + if err = c.Set(key, val); err != nil { + log.Fatalf("Failed saving metadata %q:%q: '%v'", key, val, err) + } + } + log.Println("Done writing metadata") +} + +func Create(o *options.RequestWrapper) { + o.Prep() + + gkeClient := clm.GKEClient{} + clusterOps := gkeClient.Setup(o.Request) + gkeOps := clusterOps.(*clm.GKECluster) + if err := gkeOps.Acquire(); err != nil || gkeOps.Cluster == nil { + log.Fatalf("failed acquiring GKE cluster: '%v'", err) + } + + // At this point we should have a cluster ready to run test. Need to save + // metadata so that following flow can understand the context of cluster, as + // well as for Prow usage later + writeMetaData(gkeOps.Cluster, gkeOps.Project) + + // set up kube config points to cluster + // TODO(chaodaiG): this probably should also be part of clustermanager lib + if out, err := common.StandardExec("gcloud", "beta", "container", "clusters", "get-credentials", + gkeOps.Cluster.Name, "--region", gkeOps.Cluster.Location, "--project", gkeOps.Project); err != nil { + log.Fatalf("Failed connecting to cluster: %q, '%v'", out, err) + } + if out, err := common.StandardExec("gcloud", "config", "set", "project", gkeOps.Project); err != nil { + log.Fatalf("Failed setting gcloud: %q, '%v'", out, err) + } +} diff --git a/vendor/knative.dev/pkg/testutils/clustermanager/prow-cluster-operation/actions/delete.go b/vendor/knative.dev/pkg/testutils/clustermanager/prow-cluster-operation/actions/delete.go new file mode 100644 index 00000000..6f316ec6 --- /dev/null +++ b/vendor/knative.dev/pkg/testutils/clustermanager/prow-cluster-operation/actions/delete.go @@ -0,0 +1,50 @@ +/* +Copyright 2019 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package actions + +import ( + "log" + + clm "knative.dev/pkg/testutils/clustermanager/e2e-tests" + "knative.dev/pkg/testutils/clustermanager/prow-cluster-operation/options" +) + +func Delete(o *options.RequestWrapper) { + o.Request.NeedsCleanup = true + o.Request.SkipCreation = true + + gkeClient := clm.GKEClient{} + clusterOps := gkeClient.Setup(o.Request) + gkeOps := clusterOps.(*clm.GKECluster) + if err := gkeOps.Acquire(); err != nil || gkeOps.Cluster == nil { + log.Fatalf("Failed identifying cluster for cleanup: '%v'", err) + } + log.Printf("Identified project %q and cluster %q for removal", gkeOps.Project, gkeOps.Cluster.Name) + var err error + if err = gkeOps.Delete(); err != nil { + log.Fatalf("Failed deleting cluster: '%v'", err) + } + // TODO: uncomment the lines below when previous Delete command becomes + // async operation + // // Unset context with best effort. The first command only unsets current + // // context, but doesn't delete the entry from kubeconfig, and should return it's + // // context if succeeded, which can be used by the second command to + // // delete it from kubeconfig + // if out, err := common.StandardExec("kubectl", "config", "unset", "current-context"); err != nil { + // common.StandardExec("kubectl", "config", "unset", "contexts."+string(out)) + // } +} diff --git a/vendor/knative.dev/pkg/testutils/clustermanager/prow-cluster-operation/actions/get.go b/vendor/knative.dev/pkg/testutils/clustermanager/prow-cluster-operation/actions/get.go new file mode 100644 index 00000000..fe3c47cf --- /dev/null +++ b/vendor/knative.dev/pkg/testutils/clustermanager/prow-cluster-operation/actions/get.go @@ -0,0 +1,29 @@ +/* +Copyright 2019 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package actions + +import ( + "knative.dev/pkg/testutils/clustermanager/prow-cluster-operation/options" +) + +func Get(o *options.RequestWrapper) { + o.Prep() + o.Request.SkipCreation = true + // Reuse `Create` for getting operation, so that we can reuse the same logic + // such as protected project/cluster etc. + Create(o) +} diff --git a/vendor/knative.dev/pkg/testutils/clustermanager/prow-cluster-operation/main.go b/vendor/knative.dev/pkg/testutils/clustermanager/prow-cluster-operation/main.go new file mode 100644 index 00000000..16170166 --- /dev/null +++ b/vendor/knative.dev/pkg/testutils/clustermanager/prow-cluster-operation/main.go @@ -0,0 +1,53 @@ +/* +Copyright 2019 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "flag" + "log" + + "knative.dev/pkg/testutils/clustermanager/prow-cluster-operation/actions" + "knative.dev/pkg/testutils/clustermanager/prow-cluster-operation/options" +) + +var ( + create bool + delete bool + get bool +) + +func main() { + flag.BoolVar(&create, "create", false, "Create cluster") + flag.BoolVar(&delete, "delete", false, "Delete cluster") + flag.BoolVar(&get, "get", false, "Get existing cluster from kubeconfig or gcloud") + o := options.NewRequestWrapper() + flag.Parse() + + if (create && delete) || (create && get) || (delete && get) { + log.Fatal("--create, --delete, --get are mutually exclusive") + } + switch { + case create: + actions.Create(o) + case delete: + actions.Delete(o) + case get: + actions.Get(o) + default: + log.Fatal("Must pass one of --create, --delete, --get") + } +} diff --git a/vendor/knative.dev/pkg/testutils/clustermanager/prow-cluster-operation/options/options.go b/vendor/knative.dev/pkg/testutils/clustermanager/prow-cluster-operation/options/options.go new file mode 100644 index 00000000..0eb4b660 --- /dev/null +++ b/vendor/knative.dev/pkg/testutils/clustermanager/prow-cluster-operation/options/options.go @@ -0,0 +1,61 @@ +/* +Copyright 2019 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package options + +import ( + "flag" + "strings" + + clm "knative.dev/pkg/testutils/clustermanager/e2e-tests" +) + +type RequestWrapper struct { + Request clm.GKERequest + BackupRegionsStr string + AddonsStr string + NoWait bool +} + +func NewRequestWrapper() *RequestWrapper { + rw := &RequestWrapper{ + Request: clm.GKERequest{}, + } + rw.addOptions() + return rw +} + +func (rw *RequestWrapper) Prep() { + if rw.BackupRegionsStr != "" { + rw.Request.BackupRegions = strings.Split(rw.BackupRegionsStr, ",") + } + if rw.AddonsStr != "" { + rw.Request.Addons = strings.Split(rw.AddonsStr, ",") + } +} + +func (rw *RequestWrapper) addOptions() { + flag.Int64Var(&rw.Request.MinNodes, "min-nodes", 0, "minimal number of nodes") + flag.Int64Var(&rw.Request.MaxNodes, "max-nodes", 0, "maximal number of nodes") + flag.StringVar(&rw.Request.NodeType, "node-type", "", "node type") + flag.StringVar(&rw.Request.Region, "region", "", "GCP region") + flag.StringVar(&rw.Request.Zone, "zone", "", "GCP zone") + flag.StringVar(&rw.Request.Project, "project", "", "GCP project") + flag.StringVar(&rw.Request.ClusterName, "name", "", "cluster name") + flag.StringVar(&rw.BackupRegionsStr, "backup-regions", "", "GCP regions as backup, separated by comma") + flag.StringVar(&rw.AddonsStr, "addons", "", "addons to be added, separated by comma") + flag.BoolVar(&rw.Request.SkipCreation, "skip-creation", false, "should skip creation or not") +} diff --git a/vendor/knative.dev/pkg/testutils/metahelper/client/client.go b/vendor/knative.dev/pkg/testutils/metahelper/client/client.go new file mode 100644 index 00000000..87221cb9 --- /dev/null +++ b/vendor/knative.dev/pkg/testutils/metahelper/client/client.go @@ -0,0 +1,109 @@ +/* +Copyright 2019 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package client supports various needs for running tests +package client + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "log" + "os" + "path" + + "knative.dev/pkg/test/prow" +) + +const ( + filename = "metadata.json" +) + +// client holds metadata as a string:string map, as well as path for storing +// metadata +type client struct { + MetaData map[string]string + Path string +} + +// NewClient creates a client, takes custom directory for storing `metadata.json`. +// It reads existing `metadata.json` file if it exists, otherwise creates it. +// Errors out if there is any file i/o problem other than file not exist error. +func NewClient(dir string) (*client, error) { + c := &client{ + MetaData: make(map[string]string), + } + if dir == "" { + log.Println("Getting artifacts dir from prow") + dir = prow.GetLocalArtifactsDir() + } + c.Path = path.Join(dir, filename) + if _, err := os.Stat(dir); os.IsNotExist(err) { + if err = os.MkdirAll(dir, 0777); err != nil { + return nil, fmt.Errorf("Failed to create directory: %v", err) + } + } + return c, nil +} + +// sync is shared by Get and Set, invoked at the very beginning of each, makes +// sure the file exists, and loads the content of file into c.MetaData +func (c *client) sync() error { + _, err := os.Stat(c.Path) + if os.IsNotExist(err) { + body, _ := json.Marshal(&c.MetaData) + err = ioutil.WriteFile(c.Path, body, 0777) + } else { + var body []byte + body, err = ioutil.ReadFile(c.Path) + if err == nil { + err = json.Unmarshal(body, &c.MetaData) + } + } + + return err +} + +// Set sets key:val pair, and overrides if it exists +func (c *client) Set(key, val string) error { + err := c.sync() + if err != nil { + return err + } + if oldVal, ok := c.MetaData[key]; ok { + log.Printf("Overriding meta %q:%q with new value %q", key, oldVal, val) + } + c.MetaData[key] = val + body, _ := json.Marshal(c.MetaData) + return ioutil.WriteFile(c.Path, body, 0777) +} + +// Get gets val for key +func (c *client) Get(key string) (string, error) { + if _, err := os.Stat(c.Path); err != nil && os.IsNotExist(err) { + return "", fmt.Errorf("file %q doesn't exist", c.Path) + } + var res string + err := c.sync() + if err == nil { + if val, ok := c.MetaData[key]; ok { + res = val + } else { + err = fmt.Errorf("key %q doesn't exist", key) + } + } + return res, err +} diff --git a/vendor/knative.dev/pkg/testutils/metahelper/main.go b/vendor/knative.dev/pkg/testutils/metahelper/main.go new file mode 100644 index 00000000..f6bf4e73 --- /dev/null +++ b/vendor/knative.dev/pkg/testutils/metahelper/main.go @@ -0,0 +1,66 @@ +/* +Copyright 2019 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "flag" + "fmt" + "log" + + "knative.dev/pkg/testutils/metahelper/client" +) + +var ( + getKeyOpt string + saveKeyOpt string + valOpt string +) + +func main() { + flag.StringVar(&getKeyOpt, "get", "", "get val for a key") + flag.StringVar(&saveKeyOpt, "set", "", "save val for a key, must have --val supplied") + flag.StringVar(&valOpt, "val", "", "val to be modified, only useful when --save is passed") + flag.Parse() + // Create with default path of metahelper/client, so that the path is + // consistent with all other consumers of metahelper/client that run within + // the same context of this tool + c, err := client.NewClient("") + if err != nil { + log.Fatal(err) + } + + var res string + switch { + case getKeyOpt != "" && saveKeyOpt != "": + log.Fatal("--get and --save can't be used at the same time") + case getKeyOpt != "": + gotVal, err := c.Get(getKeyOpt) + if err != nil { + log.Fatalf("Failed getting value for %q from %q: '%v'", getKeyOpt, c.Path, err) + } + res = gotVal + case saveKeyOpt != "": + if valOpt == "" { + log.Fatal("--val must be supplied when using --save") + } + log.Printf("Writing files to %s", c.Path) + if err := c.Set(saveKeyOpt, valOpt); err != nil { + log.Fatalf("Failed saving %q:%q to %q: '%v'", saveKeyOpt, valOpt, c.Path, err) + } + } + fmt.Print(res) +} diff --git a/vendor/knative.dev/pkg/websocket/connection.go b/vendor/knative.dev/pkg/websocket/connection.go index 64603925..6518e387 100644 --- a/vendor/knative.dev/pkg/websocket/connection.go +++ b/vendor/knative.dev/pkg/websocket/connection.go @@ -20,7 +20,6 @@ import ( "bytes" "encoding/gob" "errors" - "fmt" "io" "io/ioutil" "sync" @@ -125,20 +124,20 @@ func NewDurableConnection(target string, messageChan chan []byte, logger *zap.Su for { select { default: - logger.Infof("Connecting to %q", target) + logger.Infof("Connecting to %s", target) if err := c.connect(); err != nil { - logger.Errorw(fmt.Sprintf("Connecting to %q failed", target), zap.Error(err)) + logger.With(zap.Error(err)).Errorf("Connecting to %s failed", target) continue } - logger.Infof("Connected to %q", target) + logger.Debugf("Connected to %s", target) if err := c.keepalive(); err != nil { - logger.Errorw(fmt.Sprintf("Connection to %q broke down, reconnecting...", target), zap.Error(err)) + logger.With(zap.Error(err)).Errorf("Connection to %s broke down, reconnecting...", target) } if err := c.closeConnection(); err != nil { logger.Errorw("Failed to close the connection after crashing", zap.Error(err)) } case <-c.closeChan: - logger.Infof("Connection to %q is being shutdown", target) + logger.Infof("Connection to %s is being shutdown", target) return } } @@ -185,12 +184,10 @@ func newConnection(connFactory func() (rawConnection, error), messageChan chan [ // connect tries to establish a websocket connection. func (c *ManagedConnection) connect() error { - var err error - wait.ExponentialBackoff(c.connectionBackoff, func() (bool, error) { + return wait.ExponentialBackoff(c.connectionBackoff, func() (bool, error) { select { default: - var conn rawConnection - conn, err = c.connectionFactory() + conn, err := c.connectionFactory() if err != nil { return false, nil } @@ -211,12 +208,9 @@ func (c *ManagedConnection) connect() error { c.connection = conn return true, nil case <-c.closeChan: - err = errShuttingDown - return false, err + return false, errShuttingDown } }) - - return err } // keepalive keeps the connection open. diff --git a/vendor/knative.dev/test-infra/scripts/README.md b/vendor/knative.dev/test-infra/scripts/README.md index 549a540c..3b9e3a1c 100644 --- a/vendor/knative.dev/test-infra/scripts/README.md +++ b/vendor/knative.dev/test-infra/scripts/README.md @@ -61,7 +61,7 @@ This is a helper script to run the presubmit tests. To use it: the integration tests (either your custom one or the default action) and will cause the test to fail if they don't return success. -1. Call the `main()` function passing `$@` (without quotes). +1. Call the `main()` function passing `"$@"` (with quotes). Running the script without parameters, or with the `--all-tests` flag causes all tests to be executed, in the right order (i.e., build, then unit, then @@ -72,6 +72,11 @@ specific set of tests. The flag `--emit-metrics` is used to emit metrics when running the tests, and is automatically handled by the default action for integration tests (see above). +To run a specific program as a test, use the `--run-test` flag, and provide the +program as the argument. If arguments are required for the program, pass everything +as a single quotes argument. For example, `./presubmit-tests.sh --run-test +"test/my/test data"`. + The script will automatically skip all presubmit tests for PRs where all changed files are exempt of tests (e.g., a PR changing only the `OWNERS` file). @@ -99,7 +104,7 @@ function pre_integration_tests() { # We use the default integration test runner. -main $@ +main "$@" ``` ## Using the `e2e-tests.sh` helper script @@ -224,6 +229,65 @@ kubectl get pods || fail_test success ``` +## Using the `performance-tests.sh` helper script + +This is a helper script for Knative performance test scripts. In combination +with specific Prow jobs, it can automatically manage the environment for running +benchmarking jobs for each repo. To use it: + +1. Source the script. + +1. [optional] Customize GCP project settings for the benchmarks. Set the + following environment variables if the default value doesn't fit your needs: + + - `PROJECT_NAME`: GCP project name for keeping the clusters that run the + benchmarks. Defaults to `knative-performance`. + - `SERVICE_ACCOUNT_NAME`: Service account name for controlling GKE clusters + and interacting with [Mako](https://github.com/google/mako) server. It MUST have + `Kubernetes Engine Admin` and `Storage Admin` role, and be + [whitelisted](https://github.com/google/mako/blob/master/docs/ACCESS.md) by + Mako admin. Defaults to `mako-job`. + +1. [optional] Customize root path of the benchmarks. This root folder should + contain and only contain all benchmarks you want to run continuously. + Set the following environment variable if the default value doesn't fit your + needs: + + - `BENCHMARK_ROOT_PATH`: Benchmark root path, defaults to + `test/performance/benchmarks`. Each repo can decide which folder to put its + benchmarks in, and override this environment variable to be the path of + that folder. + +1. [optional] Write the `update_knative` function, which will update your + system under test (e.g. Knative Serving). + +1. [optional] Write the `update_benchmark` function, which will update the + underlying resources for the benchmark (usually Knative resources and + Kubernetes cronjobs for benchmarking). This function accepts a parameter, + which is the benchmark name in the current repo. + +1. Call the `main()` function with all parameters (e.g. `$@`). + +### Sample performance test script + +This script will update `Knative serving` and the given benchmark. + +```bash +source vendor/knative.dev/test-infra/scripts/performance-tests.sh + +function update_knative() { + echo ">> Updating serving" + ko apply -f config/ || abort "failed to apply serving" +} + +function update_benchmark() { + echo ">> Updating benchmark $1" + ko apply -f ${BENCHMARK_ROOT_PATH}/$1 || abort "failed to apply benchmark $1" +} + +main $@ +``` + ## Using the `release.sh` helper script This is a helper script for Knative release scripts. To use it: diff --git a/vendor/knative.dev/test-infra/scripts/performance-tests.sh b/vendor/knative.dev/test-infra/scripts/performance-tests.sh new file mode 100755 index 00000000..59961857 --- /dev/null +++ b/vendor/knative.dev/test-infra/scripts/performance-tests.sh @@ -0,0 +1,150 @@ +#!/bin/bash + +# Copyright 2019 The Knative Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is a helper script for Knative performance test scripts. +# See README.md for instructions on how to use it. + +source $(dirname ${BASH_SOURCE})/library.sh + +# Configurable parameters. +# If not provided, they will fall back to the default values. +readonly BENCHMARK_ROOT_PATH=${BENCHMARK_ROOT_PATH:-test/performance/benchmarks} +readonly PROJECT_NAME=${PROJECT_NAME:-knative-performance} +readonly SERVICE_ACCOUNT_NAME=${SERVICE_ACCOUNT_NAME:-mako-job} + +# Setup env vars. +readonly KO_DOCKER_REPO="gcr.io/${PROJECT_NAME}" +readonly GOOGLE_APPLICATION_CREDENTIALS="/etc/performance-test/service-account.json" +readonly GITHUB_TOKEN="/etc/performance-test/github-token" +readonly SLACK_READ_TOKEN="/etc/performance-test/slack-read-token" +readonly SLACK_WRITE_TOKEN="/etc/performance-test/slack-write-token" + +# Set up the user for cluster operations. +function setup_user() { + echo ">> Setting up user" + echo "Using gcloud project ${PROJECT_NAME}" + gcloud config set core/project ${PROJECT_NAME} + local user_name="${SERVICE_ACCOUNT_NAME}@${PROJECT_NAME}.iam.gserviceaccount.com" + echo "Using gcloud user ${user_name}" + gcloud config set core/account ${user_name} +} + +# Update resources installed on the cluster. +# Parameters: $1 - cluster name +# $2 - cluster region/zone +function update_cluster() { + # --zone option can work with both region and zone, (e.g. us-central1 and + # us-central1-a), so we don't need to add extra check here. + gcloud container clusters get-credentials $1 --zone=$2 --project=${PROJECT_NAME} || abort "failed to get cluster creds" + # Set up the configmap to run benchmarks in production + echo ">> Setting up 'prod' config-mako on cluster $1 in zone $2" + cat <> Creating secrets on cluster $1 in zone $2" + kubectl create secret generic mako-secrets \ + --from-file=robot.json=${GOOGLE_APPLICATION_CREDENTIALS} \ + --from-file=github-token=${GITHUB_TOKEN} \ + --from-file=slack-read-token=${SLACK_READ_TOKEN} \ + --from-file=slack-write-token=${SLACK_WRITE_TOKEN} + # Delete all benchmark jobs to avoid noise in the update process + echo ">> Deleting all cronjobs and jobs on cluster $1 in zone $2" + kubectl delete cronjob --all + kubectl delete job --all + + if function_exists update_knative; then + update_knative || abort "failed to update knative" + fi + # get benchmark name from the cluster name + local benchmark_name=$(get_benchmark_name $1) + if function_exists update_benchmark; then + update_benchmark ${benchmark_name} || abort "failed to update benchmark" + fi +} + +# Get benchmark name from the cluster name. +# Parameters: $1 - cluster name +function get_benchmark_name() { + # get benchmark_name by removing the prefix from cluster name, e.g. get "load-test" from "serving--load-test" + echo ${1#$REPO_NAME"--"} +} + +# Update the clusters related to the current repo. +function update_clusters() { + header "Updating all clusters for ${REPO_NAME}" + local all_clusters=$(gcloud container clusters list --project="${PROJECT_NAME}" --format="csv[no-heading](name,zone)") + echo ">> Project contains clusters:" ${all_clusters} + for cluster in ${all_clusters}; do + local name=$(echo "${cluster}" | cut -f1 -d",") + # the cluster name is prefixed with "${REPO_NAME}--", here we should only handle clusters belonged to the current repo + [[ ! ${name} =~ ^${REPO_NAME}-- ]] && continue + local zone=$(echo "${cluster}" | cut -f2 -d",") + + # Update all resources installed on the cluster + update_cluster ${name} ${zone} + done + header "Done updating all clusters" +} + +# Delete the old clusters belonged to the current repo, and recreate them with the same configuration. +function recreate_clusters() { + header "Recreating clusters for ${REPO_NAME}" + go run ${REPO_ROOT_DIR}/vendor/knative.dev/pkg/testutils/clustermanager/perf-tests \ + --recreate \ + --gcp-project=${PROJECT_NAME} --repository=${REPO_NAME} --benchmark-root=${BENCHMARK_ROOT_PATH} + header "Done recreating clusters" + # Update all clusters after they are recreated + update_clusters +} + +# Try to reconcile clusters for benchmarks in the current repo. +# This function will be run as postsubmit jobs. +function reconcile_benchmark_clusters() { + header "Reconciling clusters for ${REPO_NAME}" + go run ${REPO_ROOT_DIR}/vendor/knative.dev/pkg/testutils/clustermanager/perf-tests \ + --reconcile \ + --gcp-project=${PROJECT_NAME} --repository=${REPO_NAME} --benchmark-root=${BENCHMARK_ROOT_PATH} + header "Done reconciling clusters" + # For now, do nothing after reconciling the clusters, and the next update_clusters job will automatically + # update them. So there will be a period that the newly created clusters are being idle, and the duration + # can be as long as . +} + +# Parse flags and excute the command. +function main() { + if (( ! IS_PROW )); then + abort "this script should only be run by Prow since it needs secrets created on Prow cluster" + fi + + # Set up the user credential for cluster operations + setup_user || abort "failed to set up user" + + # Try parsing the first flag as a command. + case $1 in + --recreate-clusters) recreate_clusters ;; + --update-clusters) update_clusters ;; + --reconcile-benchmark-clusters) reconcile_benchmark_clusters ;; + *) abort "unknown command $1, must be --recreate-clusters, --update-clusters or --reconcile_benchmark_clusters" + esac + shift +} diff --git a/vendor/knative.dev/test-infra/scripts/presubmit-tests.sh b/vendor/knative.dev/test-infra/scripts/presubmit-tests.sh index e206abbe..0f95529b 100755 --- a/vendor/knative.dev/test-infra/scripts/presubmit-tests.sh +++ b/vendor/knative.dev/test-infra/scripts/presubmit-tests.sh @@ -342,7 +342,7 @@ function main() { --run-test) shift [[ $# -ge 1 ]] || abort "missing executable after --run-test" - TEST_TO_RUN=$1 + TEST_TO_RUN="$1" ;; *) abort "error: unknown option ${parameter}" ;; esac