diff --git a/Gopkg.lock b/Gopkg.lock index 97a1935a..d8785834 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -927,7 +927,7 @@ [[projects]] branch = "master" - digest = "1:ee6796d9df631ca8c5cf5617521af3086f2c67f8c941e440cd5c7a935b63da76" + digest = "1:6e410a2318808cce9fd9fcdc5c1821b96b89ad1cbe17251c9915c3c4da1b27fe" name = "knative.dev/pkg" packages = [ "apis", @@ -946,18 +946,18 @@ "metrics/metricskey", ] pruneopts = "T" - revision = "4a790dd36c6c194892fd3cdb8039667ad391e210" + revision = "d90ec6a0150ac99846556d88d5a54093418ec2a7" [[projects]] branch = "master" - digest = "1:2226e46ebce37abefef1e100a1c77cdbbff6e76bfe0f99b061abb5bdf1849a3c" + digest = "1:d65a0f4c8458b9720f97b17d6d04c9efe341298f1c3f6c37e124a6c83c716668" name = "knative.dev/test-infra" packages = [ "scripts", "tools/dep-collector", ] pruneopts = "UT" - revision = "cc2b86828e9e7d4992029981667f9cc9a69acd96" + revision = "baf7e79986726748f7b36aa8f31f8bbfaa8717b6" [[projects]] digest = "1:8730e0150dfb2b7e173890c8b9868e7a273082ef8e39f4940e3506a481cf895c" diff --git a/vendor/knative.dev/pkg/Gopkg.lock b/vendor/knative.dev/pkg/Gopkg.lock index fb1fb601..7ea7b2d1 100644 --- a/vendor/knative.dev/pkg/Gopkg.lock +++ b/vendor/knative.dev/pkg/Gopkg.lock @@ -1251,14 +1251,14 @@ [[projects]] branch = "master" - digest = "1:964a4f9e12d5021b9d550e04c36cf23feb880e286ceede1b246d5c0e43b523b9" + digest = "1:2226e46ebce37abefef1e100a1c77cdbbff6e76bfe0f99b061abb5bdf1849a3c" name = "knative.dev/test-infra" packages = [ "scripts", "tools/dep-collector", ] pruneopts = "UT" - revision = "efad44559f944cfc58dc5bca076eeb3ec20fb346" + revision = "cc2b86828e9e7d4992029981667f9cc9a69acd96" [[projects]] digest = "1:8730e0150dfb2b7e173890c8b9868e7a273082ef8e39f4940e3506a481cf895c" diff --git a/vendor/knative.dev/pkg/configmap/filter.go b/vendor/knative.dev/pkg/configmap/filter.go index 27bf13df..ed1040e2 100644 --- a/vendor/knative.dev/pkg/configmap/filter.go +++ b/vendor/knative.dev/pkg/configmap/filter.go @@ -16,7 +16,12 @@ limitations under the License. package configmap -import "reflect" +import ( + "fmt" + "reflect" + + corev1 "k8s.io/api/core/v1" +) // TypeFilter accepts instances of types to check against and returns a function transformer that would only let // the call to f through if value is assignable to any one of types of ts. Example: @@ -42,3 +47,28 @@ func TypeFilter(ts ...interface{}) func(func(string, interface{})) func(string, } } } + +// ValidateConstructor checks the type of the constructor it evaluates +// the constructor to be a function with correct signature. +// +// The expectation is for the constructor to receive a single input +// parameter of type corev1.ConfigMap as the input and return two +// values with the second value being of type error +func ValidateConstructor(constructor interface{}) error { + cType := reflect.TypeOf(constructor) + + if cType.Kind() != reflect.Func { + return fmt.Errorf("config constructor must be a function") + } + + if cType.NumIn() != 1 || cType.In(0) != reflect.TypeOf(&corev1.ConfigMap{}) { + return fmt.Errorf("config constructor must be of the type func(*k8s.io/api/core/v1/ConfigMap) (..., error)") + } + + errorType := reflect.TypeOf((*error)(nil)).Elem() + + if cType.NumOut() != 2 || !cType.Out(1).Implements(errorType) { + return fmt.Errorf("config constructor must be of the type func(*k8s.io/api/core/v1/ConfigMap) (..., error)") + } + return nil +} diff --git a/vendor/knative.dev/pkg/configmap/store.go b/vendor/knative.dev/pkg/configmap/store.go index 012d35de..825803c2 100644 --- a/vendor/knative.dev/pkg/configmap/store.go +++ b/vendor/knative.dev/pkg/configmap/store.go @@ -101,20 +101,8 @@ func NewUntypedStore( } func (s *UntypedStore) registerConfig(name string, constructor interface{}) { - cType := reflect.TypeOf(constructor) - - if cType.Kind() != reflect.Func { - panic("config constructor must be a function") - } - - if cType.NumIn() != 1 || cType.In(0) != reflect.TypeOf(&corev1.ConfigMap{}) { - panic("config constructor must be of the type func(*k8s.io/api/core/v1/ConfigMap) (..., error)") - } - - errorType := reflect.TypeOf((*error)(nil)).Elem() - - if cType.NumOut() != 2 || !cType.Out(1).Implements(errorType) { - panic("config constructor must be of the type func(*k8s.io/api/core/v1/ConfigMap) (..., error)") + if err := ValidateConstructor(constructor); err != nil { + panic(err) } s.storages[name] = &atomic.Value{} diff --git a/vendor/knative.dev/pkg/controller/controller.go b/vendor/knative.dev/pkg/controller/controller.go index eb6ba8d1..0b8be1c6 100644 --- a/vendor/knative.dev/pkg/controller/controller.go +++ b/vendor/knative.dev/pkg/controller/controller.go @@ -250,27 +250,21 @@ func (c *Impl) EnqueueLabelOfClusterScopedResource(nameLabel string) func(obj in return } - ns, name, err := cache.SplitMetaNamespaceKey(controllerKey) - if err != nil { - c.logger.Error(err) - return - } - - c.EnqueueKey(types.NamespacedName{Namespace: ns, Name: name}) + c.EnqueueKey(types.NamespacedName{Namespace: "", Name: controllerKey}) } } // EnqueueKey takes a namespace/name string and puts it onto the work queue. func (c *Impl) EnqueueKey(key types.NamespacedName) { c.WorkQueue.Add(key) - c.logger.Debugf("Adding to queue %s (depth: %d)", key.String(), c.WorkQueue.Len()) + c.logger.Debugf("Adding to queue %s (depth: %d)", safeKey(key), c.WorkQueue.Len()) } // EnqueueKeyAfter takes a namespace/name string and schedules its execution in // the work queue after given delay. func (c *Impl) EnqueueKeyAfter(key types.NamespacedName, delay time.Duration) { c.WorkQueue.AddAfter(key, delay) - c.logger.Debugf("Adding to queue %s (delay: %v, depth: %d)", key.String(), delay, c.WorkQueue.Len()) + c.logger.Debugf("Adding to queue %s (delay: %v, depth: %d)", safeKey(key), delay, c.WorkQueue.Len()) } // Run starts the controller's worker threads, the number of which is threadiness. @@ -314,9 +308,9 @@ func (c *Impl) processNextWorkItem() bool { return false } key := obj.(types.NamespacedName) - keyStr := key.String() + keyStr := safeKey(key) - c.logger.Debugf("Processing from queue %s (depth: %d)", key, c.WorkQueue.Len()) + c.logger.Debugf("Processing from queue %s (depth: %d)", safeKey(key), c.WorkQueue.Len()) startTime := time.Now() // Send the metrics for the current queue depth @@ -368,7 +362,7 @@ func (c *Impl) handleErr(err error, key types.NamespacedName) { // being processed, queue.Len==0). if !IsPermanentError(err) && !c.WorkQueue.ShuttingDown() { c.WorkQueue.AddRateLimited(key) - c.logger.Debugf("Requeuing key %s due to non-permanent error (depth: %d)", key.String(), c.WorkQueue.Len()) + c.logger.Debugf("Requeuing key %s due to non-permanent error (depth: %d)", safeKey(key), c.WorkQueue.Len()) return } @@ -508,3 +502,10 @@ func GetEventRecorder(ctx context.Context) record.EventRecorder { } return untyped.(record.EventRecorder) } + +func safeKey(key types.NamespacedName) string { + if key.Namespace == "" { + return key.Name + } + return key.String() +} diff --git a/vendor/knative.dev/pkg/injection/sharedmain/main.go b/vendor/knative.dev/pkg/injection/sharedmain/main.go index c7048850..73c154c8 100644 --- a/vendor/knative.dev/pkg/injection/sharedmain/main.go +++ b/vendor/knative.dev/pkg/injection/sharedmain/main.go @@ -25,7 +25,9 @@ import ( "os" "os/user" "path/filepath" + "time" + "go.opencensus.io/stats/view" "golang.org/x/sync/errgroup" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/rest" @@ -113,6 +115,14 @@ func MainWithConfig(ctx context.Context, component string, cfg *rest.Config, cto log.Printf("Registering %d informers", len(injection.Default.GetInformers())) log.Printf("Registering %d controllers", len(ctors)) + // Report stats on Go memory usage every 30 seconds. + msp := metrics.NewMemStatsAll() + msp.Start(ctx, 30*time.Second) + + if err := view.Register(msp.DefaultViews()...); err != nil { + log.Fatalf("Error exporting go memstats view: %v", err) + } + // Adjust our client's rate limits based on the number of controller's we are running. cfg.QPS = float32(len(ctors)) * rest.DefaultQPS cfg.Burst = len(ctors) * rest.DefaultBurst diff --git a/vendor/knative.dev/pkg/logging/testing/util.go b/vendor/knative.dev/pkg/logging/testing/util.go index 29daf993..6773a10f 100644 --- a/vendor/knative.dev/pkg/logging/testing/util.go +++ b/vendor/knative.dev/pkg/logging/testing/util.go @@ -18,7 +18,6 @@ package testing import ( "context" - "sync" "testing" "go.uber.org/zap" @@ -27,40 +26,19 @@ import ( "knative.dev/pkg/logging" ) -var ( - loggers = make(map[string]*zap.SugaredLogger) - m sync.Mutex -) - // TestLogger gets a logger to use in unit and end to end tests func TestLogger(t *testing.T) *zap.SugaredLogger { - m.Lock() - defer m.Unlock() - - logger, ok := loggers[t.Name()] - - if ok { - return logger - } - opts := zaptest.WrapOptions( zap.AddCaller(), zap.Development(), ) - logger = zaptest.NewLogger(t, opts).Sugar().Named(t.Name()) - loggers[t.Name()] = logger - - return logger + return zaptest.NewLogger(t, opts).Sugar() } // ClearAll removes all the testing loggers. -// `go test -count=X` executes runs in the same process, thus the map -// persists between the runs, but the `t` will no longer be valid and will -// cause a panic deep inside testing code. -func ClearAll() { - loggers = make(map[string]*zap.SugaredLogger) -} +// TODO(taragu) remove this after removing all ClearAll() calls from serving and eventing +func ClearAll() {} // TestContextWithLogger returns a context with a logger to be used in tests func TestContextWithLogger(t *testing.T) context.Context { diff --git a/vendor/knative.dev/pkg/metrics/memstats.go b/vendor/knative.dev/pkg/metrics/memstats.go new file mode 100644 index 00000000..51c1ede1 --- /dev/null +++ b/vendor/knative.dev/pkg/metrics/memstats.go @@ -0,0 +1,539 @@ +/* +Copyright 2019 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package metrics + +import ( + "context" + "runtime" + "time" + + "go.opencensus.io/stats" + "go.opencensus.io/stats/view" +) + +// NewMemStatsAll creates a new MemStatsProvider with stats for all of the +// supported Go runtime.MemStat fields. +func NewMemStatsAll() *MemStatsProvider { + return &MemStatsProvider{ + Alloc: stats.Int64( + "go_alloc", + "The number of bytes of allocated heap objects.", + stats.UnitNone, + ), + TotalAlloc: stats.Int64( + "go_total_alloc", + "The cumulative bytes allocated for heap objects.", + stats.UnitNone, + ), + Sys: stats.Int64( + "go_sys", + "The total bytes of memory obtained from the OS.", + stats.UnitNone, + ), + Lookups: stats.Int64( + "go_lookups", + "The number of pointer lookups performed by the runtime.", + stats.UnitNone, + ), + Mallocs: stats.Int64( + "go_mallocs", + "The cumulative count of heap objects allocated.", + stats.UnitNone, + ), + Frees: stats.Int64( + "go_frees", + "The cumulative count of heap objects freed.", + stats.UnitNone, + ), + HeapAlloc: stats.Int64( + "go_heap_alloc", + "The number of bytes of allocated heap objects.", + stats.UnitNone, + ), + HeapSys: stats.Int64( + "go_heap_sys", + "The number of bytes of heap memory obtained from the OS.", + stats.UnitNone, + ), + HeapIdle: stats.Int64( + "go_heap_idle", + "The number of bytes in idle (unused) spans.", + stats.UnitNone, + ), + HeapInuse: stats.Int64( + "go_heap_in_use", + "The number of bytes in in-use spans.", + stats.UnitNone, + ), + HeapReleased: stats.Int64( + "go_heap_released", + "The number of bytes of physical memory returned to the OS.", + stats.UnitNone, + ), + HeapObjects: stats.Int64( + "go_heap_objects", + "The number of allocated heap objects.", + stats.UnitNone, + ), + StackInuse: stats.Int64( + "go_stack_in_use", + "The number of bytes in stack spans.", + stats.UnitNone, + ), + StackSys: stats.Int64( + "go_stack_sys", + "The number of bytes of stack memory obtained from the OS.", + stats.UnitNone, + ), + MSpanInuse: stats.Int64( + "go_mspan_in_use", + "The number of bytes of allocated mspan structures.", + stats.UnitNone, + ), + MSpanSys: stats.Int64( + "go_mspan_sys", + "The number of bytes of memory obtained from the OS for mspan structures.", + stats.UnitNone, + ), + MCacheInuse: stats.Int64( + "go_mcache_in_use", + "The number of bytes of allocated mcache structures.", + stats.UnitNone, + ), + MCacheSys: stats.Int64( + "go_mcache_sys", + "The number of bytes of memory obtained from the OS for mcache structures.", + stats.UnitNone, + ), + BuckHashSys: stats.Int64( + "go_bucket_hash_sys", + "The number of bytes of memory in profiling bucket hash tables.", + stats.UnitNone, + ), + GCSys: stats.Int64( + "go_gc_sys", + "The number of bytes of memory in garbage collection metadata.", + stats.UnitNone, + ), + OtherSys: stats.Int64( + "go_other_sys", + "The number of bytes of memory in miscellaneous off-heap runtime allocations.", + stats.UnitNone, + ), + NextGC: stats.Int64( + "go_next_gc", + "The target heap size of the next GC cycle.", + stats.UnitNone, + ), + LastGC: stats.Int64( + "go_last_gc", + "The time the last garbage collection finished, as nanoseconds since 1970 (the UNIX epoch).", + "ns", + ), + PauseTotalNs: stats.Int64( + "go_total_gc_pause_ns", + "The cumulative nanoseconds in GC stop-the-world pauses since the program started.", + "ns", + ), + NumGC: stats.Int64( + "go_num_gc", + "The number of completed GC cycles.", + stats.UnitNone, + ), + NumForcedGC: stats.Int64( + "go_num_forced_gc", + "The number of GC cycles that were forced by the application calling the GC function.", + stats.UnitNone, + ), + GCCPUFraction: stats.Float64( + "go_gc_cpu_fraction", + "The fraction of this program's available CPU time used by the GC since the program started.", + stats.UnitNone, + ), + } +} + +// MemStatsProvider is used to expose metrics based on Go's runtime.MemStats. +// The fields below (and their comments) are a filtered list taken from +// Go's runtime.MemStats. +type MemStatsProvider struct { + // Alloc is bytes of allocated heap objects. + // + // This is the same as HeapAlloc (see below). + Alloc *stats.Int64Measure + + // TotalAlloc is cumulative bytes allocated for heap objects. + // + // TotalAlloc increases as heap objects are allocated, but + // unlike Alloc and HeapAlloc, it does not decrease when + // objects are freed. + TotalAlloc *stats.Int64Measure + + // Sys is the total bytes of memory obtained from the OS. + // + // Sys is the sum of the XSys fields below. Sys measures the + // virtual address space reserved by the Go runtime for the + // heap, stacks, and other internal data structures. It's + // likely that not all of the virtual address space is backed + // by physical memory at any given moment, though in general + // it all was at some point. + Sys *stats.Int64Measure + + // Lookups is the number of pointer lookups performed by the + // runtime. + // + // This is primarily useful for debugging runtime internals. + Lookups *stats.Int64Measure + + // Mallocs is the cumulative count of heap objects allocated. + // The number of live objects is Mallocs - Frees. + Mallocs *stats.Int64Measure + + // Frees is the cumulative count of heap objects freed. + Frees *stats.Int64Measure + + // HeapAlloc is bytes of allocated heap objects. + // + // "Allocated" heap objects include all reachable objects, as + // well as unreachable objects that the garbage collector has + // not yet freed. Specifically, HeapAlloc increases as heap + // objects are allocated and decreases as the heap is swept + // and unreachable objects are freed. Sweeping occurs + // incrementally between GC cycles, so these two processes + // occur simultaneously, and as a result HeapAlloc tends to + // change smoothly (in contrast with the sawtooth that is + // typical of stop-the-world garbage collectors). + HeapAlloc *stats.Int64Measure + + // HeapSys is bytes of heap memory obtained from the OS. + // + // HeapSys measures the amount of virtual address space + // reserved for the heap. This includes virtual address space + // that has been reserved but not yet used, which consumes no + // physical memory, but tends to be small, as well as virtual + // address space for which the physical memory has been + // returned to the OS after it became unused (see HeapReleased + // for a measure of the latter). + // + // HeapSys estimates the largest size the heap has had. + HeapSys *stats.Int64Measure + + // HeapIdle is bytes in idle (unused) spans. + // + // Idle spans have no objects in them. These spans could be + // (and may already have been) returned to the OS, or they can + // be reused for heap allocations, or they can be reused as + // stack memory. + // + // HeapIdle minus HeapReleased estimates the amount of memory + // that could be returned to the OS, but is being retained by + // the runtime so it can grow the heap without requesting more + // memory from the OS. If this difference is significantly + // larger than the heap size, it indicates there was a recent + // transient spike in live heap size. + HeapIdle *stats.Int64Measure + + // HeapInuse is bytes in in-use spans. + // + // In-use spans have at least one object in them. These spans + // can only be used for other objects of roughly the same + // size. + // + // HeapInuse minus HeapAlloc estimates the amount of memory + // that has been dedicated to particular size classes, but is + // not currently being used. This is an upper bound on + // fragmentation, but in general this memory can be reused + // efficiently. + HeapInuse *stats.Int64Measure + + // HeapReleased is bytes of physical memory returned to the OS. + // + // This counts heap memory from idle spans that was returned + // to the OS and has not yet been reacquired for the heap. + HeapReleased *stats.Int64Measure + + // HeapObjects is the number of allocated heap objects. + // + // Like HeapAlloc, this increases as objects are allocated and + // decreases as the heap is swept and unreachable objects are + // freed. + HeapObjects *stats.Int64Measure + + // StackInuse is bytes in stack spans. + // + // In-use stack spans have at least one stack in them. These + // spans can only be used for other stacks of the same size. + // + // There is no StackIdle because unused stack spans are + // returned to the heap (and hence counted toward HeapIdle). + StackInuse *stats.Int64Measure + + // StackSys is bytes of stack memory obtained from the OS. + // + // StackSys is StackInuse, plus any memory obtained directly + // from the OS for OS thread stacks (which should be minimal). + StackSys *stats.Int64Measure + + // MSpanInuse is bytes of allocated mspan structures. + MSpanInuse *stats.Int64Measure + + // MSpanSys is bytes of memory obtained from the OS for mspan + // structures. + MSpanSys *stats.Int64Measure + + // MCacheInuse is bytes of allocated mcache structures. + MCacheInuse *stats.Int64Measure + + // MCacheSys is bytes of memory obtained from the OS for + // mcache structures. + MCacheSys *stats.Int64Measure + + // BuckHashSys is bytes of memory in profiling bucket hash tables. + BuckHashSys *stats.Int64Measure + + // GCSys is bytes of memory in garbage collection metadata. + GCSys *stats.Int64Measure + + // OtherSys is bytes of memory in miscellaneous off-heap + // runtime allocations. + OtherSys *stats.Int64Measure + + // NextGC is the target heap size of the next GC cycle. + // + // The garbage collector's goal is to keep HeapAlloc ≤ NextGC. + // At the end of each GC cycle, the target for the next cycle + // is computed based on the amount of reachable data and the + // value of GOGC. + NextGC *stats.Int64Measure + + // LastGC is the time the last garbage collection finished, as + // nanoseconds since 1970 (the UNIX epoch). + LastGC *stats.Int64Measure + + // PauseTotalNs is the cumulative nanoseconds in GC + // stop-the-world pauses since the program started. + // + // During a stop-the-world pause, all goroutines are paused + // and only the garbage collector can run. + PauseTotalNs *stats.Int64Measure + + // NumGC is the number of completed GC cycles. + NumGC *stats.Int64Measure + + // NumForcedGC is the number of GC cycles that were forced by + // the application calling the GC function. + NumForcedGC *stats.Int64Measure + + // GCCPUFraction is the fraction of this program's available + // CPU time used by the GC since the program started. + // + // GCCPUFraction is expressed as a number between 0 and 1, + // where 0 means GC has consumed none of this program's CPU. A + // program's available CPU time is defined as the integral of + // GOMAXPROCS since the program started. That is, if + // GOMAXPROCS is 2 and a program has been running for 10 + // seconds, its "available CPU" is 20 seconds. GCCPUFraction + // does not include CPU time used for write barrier activity. + // + // This is the same as the fraction of CPU reported by + // GODEBUG=gctrace=1. + GCCPUFraction *stats.Float64Measure +} + +// Start initiates a Go routine that starts pushing metrics into +// the provided measures. +func (msp *MemStatsProvider) Start(ctx context.Context, period time.Duration) { + go func() { + ticker := time.NewTicker(period) + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + ms := runtime.MemStats{} + runtime.ReadMemStats(&ms) + if msp.Alloc != nil { + Record(ctx, msp.Alloc.M(int64(ms.Alloc))) + } + if msp.TotalAlloc != nil { + Record(ctx, msp.TotalAlloc.M(int64(ms.TotalAlloc))) + } + if msp.Sys != nil { + Record(ctx, msp.Sys.M(int64(ms.Sys))) + } + if msp.Lookups != nil { + Record(ctx, msp.Lookups.M(int64(ms.Lookups))) + } + if msp.Mallocs != nil { + Record(ctx, msp.Mallocs.M(int64(ms.Mallocs))) + } + if msp.Frees != nil { + Record(ctx, msp.Frees.M(int64(ms.Frees))) + } + if msp.HeapAlloc != nil { + Record(ctx, msp.HeapAlloc.M(int64(ms.HeapAlloc))) + } + if msp.HeapSys != nil { + Record(ctx, msp.HeapSys.M(int64(ms.HeapSys))) + } + if msp.HeapIdle != nil { + Record(ctx, msp.HeapIdle.M(int64(ms.HeapIdle))) + } + if msp.HeapInuse != nil { + Record(ctx, msp.HeapInuse.M(int64(ms.HeapInuse))) + } + if msp.HeapReleased != nil { + Record(ctx, msp.HeapReleased.M(int64(ms.HeapReleased))) + } + if msp.HeapObjects != nil { + Record(ctx, msp.HeapObjects.M(int64(ms.HeapObjects))) + } + if msp.StackInuse != nil { + Record(ctx, msp.StackInuse.M(int64(ms.StackInuse))) + } + if msp.StackSys != nil { + Record(ctx, msp.StackSys.M(int64(ms.StackSys))) + } + if msp.MSpanInuse != nil { + Record(ctx, msp.MSpanInuse.M(int64(ms.MSpanInuse))) + } + if msp.MSpanSys != nil { + Record(ctx, msp.MSpanSys.M(int64(ms.MSpanSys))) + } + if msp.MCacheInuse != nil { + Record(ctx, msp.MCacheInuse.M(int64(ms.MCacheInuse))) + } + if msp.MCacheSys != nil { + Record(ctx, msp.MCacheSys.M(int64(ms.MCacheSys))) + } + if msp.BuckHashSys != nil { + Record(ctx, msp.BuckHashSys.M(int64(ms.BuckHashSys))) + } + if msp.GCSys != nil { + Record(ctx, msp.GCSys.M(int64(ms.GCSys))) + } + if msp.OtherSys != nil { + Record(ctx, msp.OtherSys.M(int64(ms.OtherSys))) + } + if msp.NextGC != nil { + Record(ctx, msp.NextGC.M(int64(ms.NextGC))) + } + if msp.LastGC != nil { + Record(ctx, msp.LastGC.M(int64(ms.LastGC))) + } + if msp.PauseTotalNs != nil { + Record(ctx, msp.PauseTotalNs.M(int64(ms.PauseTotalNs))) + } + if msp.NumGC != nil { + Record(ctx, msp.NumGC.M(int64(ms.NumGC))) + } + if msp.NumForcedGC != nil { + Record(ctx, msp.NumForcedGC.M(int64(ms.NumForcedGC))) + } + if msp.GCCPUFraction != nil { + Record(ctx, msp.GCCPUFraction.M(ms.GCCPUFraction)) + } + } + } + }() +} + +// DefaultViews returns a list of views suitable for passing to view.Register +func (msp *MemStatsProvider) DefaultViews() (views []*view.View) { + if m := msp.Alloc; m != nil { + views = append(views, measureView(m, view.LastValue())) + } + if m := msp.TotalAlloc; m != nil { + views = append(views, measureView(m, view.LastValue())) + } + if m := msp.Sys; m != nil { + views = append(views, measureView(m, view.LastValue())) + } + if m := msp.Lookups; m != nil { + views = append(views, measureView(m, view.LastValue())) + } + if m := msp.Mallocs; m != nil { + views = append(views, measureView(m, view.LastValue())) + } + if m := msp.Frees; m != nil { + views = append(views, measureView(m, view.LastValue())) + } + if m := msp.HeapAlloc; m != nil { + views = append(views, measureView(m, view.LastValue())) + } + if m := msp.HeapSys; m != nil { + views = append(views, measureView(m, view.LastValue())) + } + if m := msp.HeapIdle; m != nil { + views = append(views, measureView(m, view.LastValue())) + } + if m := msp.HeapInuse; m != nil { + views = append(views, measureView(m, view.LastValue())) + } + if m := msp.HeapReleased; m != nil { + views = append(views, measureView(m, view.LastValue())) + } + if m := msp.HeapObjects; m != nil { + views = append(views, measureView(m, view.LastValue())) + } + if m := msp.StackInuse; m != nil { + views = append(views, measureView(m, view.LastValue())) + } + if m := msp.StackSys; m != nil { + views = append(views, measureView(m, view.LastValue())) + } + if m := msp.MSpanInuse; m != nil { + views = append(views, measureView(m, view.LastValue())) + } + if m := msp.MSpanSys; m != nil { + views = append(views, measureView(m, view.LastValue())) + } + if m := msp.MCacheInuse; m != nil { + views = append(views, measureView(m, view.LastValue())) + } + if m := msp.MCacheSys; m != nil { + views = append(views, measureView(m, view.LastValue())) + } + if m := msp.BuckHashSys; m != nil { + views = append(views, measureView(m, view.LastValue())) + } + if m := msp.GCSys; m != nil { + views = append(views, measureView(m, view.LastValue())) + } + if m := msp.OtherSys; m != nil { + views = append(views, measureView(m, view.LastValue())) + } + if m := msp.NextGC; m != nil { + views = append(views, measureView(m, view.LastValue())) + } + if m := msp.LastGC; m != nil { + views = append(views, measureView(m, view.LastValue())) + } + if m := msp.PauseTotalNs; m != nil { + views = append(views, measureView(m, view.LastValue())) + } + if m := msp.NumGC; m != nil { + views = append(views, measureView(m, view.LastValue())) + } + if m := msp.NumForcedGC; m != nil { + views = append(views, measureView(m, view.LastValue())) + } + if m := msp.GCCPUFraction; m != nil { + views = append(views, measureView(m, view.LastValue())) + } + return +} diff --git a/vendor/knative.dev/pkg/resolver/addressable_resolver.go b/vendor/knative.dev/pkg/resolver/addressable_resolver.go index 5bb27f4b..dec16dac 100644 --- a/vendor/knative.dev/pkg/resolver/addressable_resolver.go +++ b/vendor/knative.dev/pkg/resolver/addressable_resolver.go @@ -24,6 +24,7 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/types" "knative.dev/pkg/apis" pkgapisduck "knative.dev/pkg/apis/duck" @@ -43,7 +44,7 @@ type URIResolver struct { } // NewURIResolver constructs a new URIResolver with context and a callback passed to the URIResolver's tracker. -func NewURIResolver(ctx context.Context, callback func(string)) *URIResolver { +func NewURIResolver(ctx context.Context, callback func(types.NamespacedName)) *URIResolver { ret := &URIResolver{} ret.tracker = tracker.New(callback, controller.GetTrackerLease(ctx)) diff --git a/vendor/knative.dev/pkg/signals/signal_posix.go b/vendor/knative.dev/pkg/signals/signal_posix.go index b8e75955..b3537d0e 100644 --- a/vendor/knative.dev/pkg/signals/signal_posix.go +++ b/vendor/knative.dev/pkg/signals/signal_posix.go @@ -23,4 +23,4 @@ import ( "syscall" ) -var shutdownSignals = []os.Signal{os.Interrupt, syscall.SIGTERM, syscall.SIGPIPE} +var shutdownSignals = []os.Signal{os.Interrupt, syscall.SIGTERM} diff --git a/vendor/knative.dev/pkg/testutils/clustermanager/gke.go b/vendor/knative.dev/pkg/testutils/clustermanager/gke.go index 7db41b9a..4c881fe8 100644 --- a/vendor/knative.dev/pkg/testutils/clustermanager/gke.go +++ b/vendor/knative.dev/pkg/testutils/clustermanager/gke.go @@ -263,7 +263,12 @@ func (gc *GKECluster) Acquire() error { // Installing addons after cluster creation takes at least 5 // minutes, so install addons as part of cluster creation, which // doesn't seem to add much time on top of cluster creation - AddonsConfig: gc.getAddonsConfig(), + AddonsConfig: gc.getAddonsConfig(), + // Equivalent to --enable-basic-auth, so that user:pass can be + // later on retrieved for setting up cluster roles. Use the + // default username from gcloud command, the password will be + // automatically generated by GKE SDK + MasterAuth: &container.MasterAuth{Username: "admin"}, InitialNodeCount: gc.Request.MinNodes, NodeConfig: &container.NodeConfig{ MachineType: gc.Request.NodeType, diff --git a/vendor/knative.dev/pkg/tracker/enqueue.go b/vendor/knative.dev/pkg/tracker/enqueue.go index 84973e93..c6357e6d 100644 --- a/vendor/knative.dev/pkg/tracker/enqueue.go +++ b/vendor/knative.dev/pkg/tracker/enqueue.go @@ -24,8 +24,8 @@ import ( "time" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/validation" - "k8s.io/client-go/tools/cache" "knative.dev/pkg/kmeta" ) @@ -38,7 +38,7 @@ import ( // When OnChanged is called by the informer for a particular // GroupVersionKind, the provided callback is called with the "key" // of each object actively watching the changed object. -func New(callback func(string), lease time.Duration) Interface { +func New(callback func(types.NamespacedName), lease time.Duration) Interface { return &impl{ leaseDuration: lease, cb: callback, @@ -55,14 +55,14 @@ type impl struct { // before having to renew the lease. leaseDuration time.Duration - cb func(string) + cb func(types.NamespacedName) } // Check that impl implements Interface. var _ Interface = (*impl)(nil) // set is a map from keys to expirations -type set map[string]time.Time +type set map[types.NamespacedName]time.Time // Track implements Interface. func (i *impl) Track(ref corev1.ObjectReference, obj interface{}) error { @@ -83,11 +83,13 @@ func (i *impl) Track(ref corev1.ObjectReference, obj interface{}) error { return fmt.Errorf("invalid ObjectReference:\n%s", strings.Join(fieldErrors, "\n")) } - key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(obj) + object, err := kmeta.DeletionHandlingAccessor(obj) if err != nil { return err } + key := types.NamespacedName{Namespace: object.GetNamespace(), Name: object.GetName()} + i.m.Lock() defer i.m.Unlock() if i.mapping == nil { diff --git a/vendor/knative.dev/pkg/webhook/config_validation_controller.go b/vendor/knative.dev/pkg/webhook/config_validation_controller.go new file mode 100644 index 00000000..ea853b72 --- /dev/null +++ b/vendor/knative.dev/pkg/webhook/config_validation_controller.go @@ -0,0 +1,217 @@ +/* +Copyright 2019 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package webhook + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "reflect" + "strings" + + "github.com/markbates/inflect" + admissionv1beta1 "k8s.io/api/admission/v1beta1" + admissionregistrationv1beta1 "k8s.io/api/admissionregistration/v1beta1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/kubernetes" + + "knative.dev/pkg/configmap" + "knative.dev/pkg/kmp" + "knative.dev/pkg/logging" +) + +// ConfigValidationController implements the AdmissionController for ConfigMaps +type ConfigValidationController struct { + constructors map[string]reflect.Value + options ControllerOptions +} + +// NewConfigValidationController constructs a ConfigValidationController +func NewConfigValidationController( + constructors configmap.Constructors, + opts ControllerOptions) AdmissionController { + cfgValidations := &ConfigValidationController{ + constructors: make(map[string]reflect.Value), + options: opts, + } + + for configName, constructor := range constructors { + cfgValidations.registerConfig(configName, constructor) + } + + return cfgValidations +} + +func (ac *ConfigValidationController) Admit(ctx context.Context, request *admissionv1beta1.AdmissionRequest) *admissionv1beta1.AdmissionResponse { + logger := logging.FromContext(ctx) + switch request.Operation { + case admissionv1beta1.Create, admissionv1beta1.Update: + default: + logger.Infof("Unhandled webhook operation, letting it through %v", request.Operation) + return &admissionv1beta1.AdmissionResponse{Allowed: true} + } + + if err := ac.validate(ctx, request); err != nil { + return makeErrorStatus("validation failed: %v", err) + } + + return &admissionv1beta1.AdmissionResponse{ + Allowed: true, + } +} + +func (ac *ConfigValidationController) Register(ctx context.Context, kubeClient kubernetes.Interface, caCert []byte) error { + client := kubeClient.AdmissionregistrationV1beta1().ValidatingWebhookConfigurations() + logger := logging.FromContext(ctx) + failurePolicy := admissionregistrationv1beta1.Fail + + resourceGVK := corev1.SchemeGroupVersion.WithKind("ConfigMap") + var rules []admissionregistrationv1beta1.RuleWithOperations + plural := strings.ToLower(inflect.Pluralize(resourceGVK.Kind)) + + ruleScope := admissionregistrationv1beta1.NamespacedScope + rules = append(rules, admissionregistrationv1beta1.RuleWithOperations{ + Operations: []admissionregistrationv1beta1.OperationType{ + admissionregistrationv1beta1.Create, + admissionregistrationv1beta1.Update, + }, + Rule: admissionregistrationv1beta1.Rule{ + APIGroups: []string{resourceGVK.Group}, + APIVersions: []string{resourceGVK.Version}, + Resources: []string{plural + "/*"}, + Scope: &ruleScope, + }, + }) + + webhook := &admissionregistrationv1beta1.ValidatingWebhookConfiguration{ + ObjectMeta: metav1.ObjectMeta{ + Name: ac.options.ConfigValidationWebhookName, + }, + Webhooks: []admissionregistrationv1beta1.ValidatingWebhook{{ + Name: ac.options.ConfigValidationWebhookName, + Rules: rules, + ClientConfig: admissionregistrationv1beta1.WebhookClientConfig{ + Service: &admissionregistrationv1beta1.ServiceReference{ + Namespace: ac.options.Namespace, + Name: ac.options.ServiceName, + Path: &ac.options.ConfigValidationControllerPath, + }, + CABundle: caCert, + }, + NamespaceSelector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{{ + Key: ac.options.ConfigValidationNamespaceLabel, + Operator: metav1.LabelSelectorOpExists, + }}, + }, + FailurePolicy: &failurePolicy, + }}, + } + + // Set the owner to our deployment. + deployment, err := kubeClient.AppsV1().Deployments(ac.options.Namespace).Get(ac.options.DeploymentName, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("failed to fetch our deployment: %v", err) + } + deploymentRef := metav1.NewControllerRef(deployment, deploymentKind) + webhook.OwnerReferences = append(webhook.OwnerReferences, *deploymentRef) + + // Try to create the webhook and if it already exists validate webhook rules. + _, err = client.Create(webhook) + if err != nil { + if !apierrors.IsAlreadyExists(err) { + return fmt.Errorf("failed to create a webhook: %v", err) + } + logger.Info("Webhook already exists") + configuredWebhook, err := client.Get(ac.options.ConfigValidationWebhookName, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("error retrieving webhook: %v", err) + } + if ok, err := kmp.SafeEqual(configuredWebhook.Webhooks, webhook.Webhooks); err != nil { + return fmt.Errorf("error diffing webhooks: %v", err) + } else if !ok { + logger.Info("Updating webhook") + // Set the ResourceVersion as required by update. + webhook.ObjectMeta.ResourceVersion = configuredWebhook.ObjectMeta.ResourceVersion + if _, err := client.Update(webhook); err != nil { + return fmt.Errorf("failed to update webhook: %s", err) + } + } else { + logger.Info("Webhook is already valid") + } + } else { + logger.Info("Created a webhook") + } + + return nil +} + +func (ac *ConfigValidationController) validate(ctx context.Context, req *admissionv1beta1.AdmissionRequest) error { + logger := logging.FromContext(ctx) + kind := req.Kind + newBytes := req.Object.Raw + + // Why, oh why are these different types... + gvk := schema.GroupVersionKind{ + Group: kind.Group, + Version: kind.Version, + Kind: kind.Kind, + } + + resourceGVK := corev1.SchemeGroupVersion.WithKind("ConfigMap") + if gvk != resourceGVK { + logger.Errorf("Unhandled kind: %v", gvk) + return fmt.Errorf("unhandled kind: %v", gvk) + } + + var newObj corev1.ConfigMap + if len(newBytes) != 0 { + newDecoder := json.NewDecoder(bytes.NewBuffer(newBytes)) + if err := newDecoder.Decode(&newObj); err != nil { + return fmt.Errorf("cannot decode incoming new object: %v", err) + } + } + + var err error + if constructor, ok := ac.constructors[newObj.Name]; ok { + + inputs := []reflect.Value{ + reflect.ValueOf(&newObj), + } + + outputs := constructor.Call(inputs) + errVal := outputs[1] + + if !errVal.IsNil() { + err = errVal.Interface().(error) + } + } + + return err +} + +func (ac *ConfigValidationController) registerConfig(name string, constructor interface{}) { + if err := configmap.ValidateConstructor(constructor); err != nil { + panic(err) + } + + ac.constructors[name] = reflect.ValueOf(constructor) +} diff --git a/vendor/knative.dev/pkg/webhook/resource_admission_controller.go b/vendor/knative.dev/pkg/webhook/resource_admission_controller.go index 93e12794..cbfacf37 100644 --- a/vendor/knative.dev/pkg/webhook/resource_admission_controller.go +++ b/vendor/knative.dev/pkg/webhook/resource_admission_controller.go @@ -30,9 +30,9 @@ import ( admissionv1beta1 "k8s.io/api/admission/v1beta1" admissionregistrationv1beta1 "k8s.io/api/admissionregistration/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/kubernetes" diff --git a/vendor/knative.dev/pkg/webhook/webhook.go b/vendor/knative.dev/pkg/webhook/webhook.go index 817198c2..f0d7d5e2 100644 --- a/vendor/knative.dev/pkg/webhook/webhook.go +++ b/vendor/knative.dev/pkg/webhook/webhook.go @@ -56,6 +56,10 @@ type ControllerOptions struct { // mutations before they get stored in the storage. ResourceMutatingWebhookName string + // ConfigValidationWebhookName is the name of the webhook we create to handle + // mutations before they get stored in the storage. + ConfigValidationWebhookName string + // ServiceName is the service name of the webhook. ServiceName string @@ -95,6 +99,13 @@ type ControllerOptions struct { // Service path for ResourceAdmissionController webhook // Default is "/" for backward compatibility and is set by the constructor ResourceAdmissionControllerPath string + + // Service path for ConfigValidationController webhook + // Default is "/config-validation" and is set by the constructor + ConfigValidationControllerPath string + + // NamespaceLabel is the label for the Namespace we bind ConfigValidationController to + ConfigValidationNamespaceLabel string } // AdmissionController provides the interface for different admission controllers diff --git a/vendor/knative.dev/test-infra/scripts/presubmit-tests.sh b/vendor/knative.dev/test-infra/scripts/presubmit-tests.sh index 205f39f4..e206abbe 100755 --- a/vendor/knative.dev/test-infra/scripts/presubmit-tests.sh +++ b/vendor/knative.dev/test-infra/scripts/presubmit-tests.sh @@ -49,7 +49,7 @@ function pr_only_contains() { # List changed files in the current PR. # This is implemented as a function so it can be mocked in unit tests. function list_changed_files() { - /workspace/githubhelper -list-changed-files + /workspace/githubhelper -list-changed-files -github-token /etc/repoview-token/token } # Initialize flags and context for presubmit tests: @@ -304,7 +304,7 @@ function main() { go version echo ">> git version" git version - echo ">> ko built from commit" + echo ">> ko version" [[ -f /ko_version ]] && cat /ko_version || echo "unknown" echo ">> bazel version" [[ -f /bazel_version ]] && cat /bazel_version || echo "unknown" diff --git a/vendor/knative.dev/test-infra/scripts/release.sh b/vendor/knative.dev/test-infra/scripts/release.sh index ab73895e..6c5a8c38 100755 --- a/vendor/knative.dev/test-infra/scripts/release.sh +++ b/vendor/knative.dev/test-infra/scripts/release.sh @@ -165,8 +165,8 @@ function prepare_auto_release() { PUBLISH_RELEASE=1 git fetch --all || abort "error fetching branches/tags from remote" - local tags="$(git tag | cut -d 'v' -f2 | cut -d '.' -f1-2 | sort | uniq)" - local branches="$( { (git branch -r | grep upstream/release-) ; (git branch | grep release-); } | cut -d '-' -f2 | sort | uniq)" + local tags="$(git tag | cut -d 'v' -f2 | cut -d '.' -f1-2 | sort -V | uniq)" + local branches="$( { (git branch -r | grep upstream/release-) ; (git branch | grep release-); } | cut -d '-' -f2 | sort -V | uniq)" echo "Versions released (from tags): [" ${tags} "]" echo "Versions released (from branches): [" ${branches} "]"