From efe31127a812420eed46520c84fd4aa5167e64cc Mon Sep 17 00:00:00 2001 From: Matt Moore Date: Mon, 9 Dec 2019 08:15:45 -0800 Subject: [PATCH] Auto-update dependencies (#155) Produced via: `dep ensure -update knative.dev/test-infra knative.dev/pkg` /assign n3wscott --- Gopkg.lock | 8 +- vendor/knative.dev/pkg/Gopkg.lock | 5 +- vendor/knative.dev/pkg/kmeta/names.go | 5 +- .../knative.dev/pkg/network/error_handler.go | 43 ++++ vendor/knative.dev/pkg/network/h2c.go | 54 +++++ vendor/knative.dev/pkg/network/network.go | 45 ++++ vendor/knative.dev/pkg/network/prober/doc.go | 18 ++ .../knative.dev/pkg/network/prober/prober.go | 192 ++++++++++++++++++ vendor/knative.dev/pkg/network/transports.go | 120 +++++++++++ .../knative.dev/test-infra/scripts/library.sh | 23 +++ 10 files changed, 507 insertions(+), 6 deletions(-) create mode 100644 vendor/knative.dev/pkg/network/error_handler.go create mode 100644 vendor/knative.dev/pkg/network/h2c.go create mode 100644 vendor/knative.dev/pkg/network/network.go create mode 100644 vendor/knative.dev/pkg/network/prober/doc.go create mode 100644 vendor/knative.dev/pkg/network/prober/prober.go create mode 100644 vendor/knative.dev/pkg/network/transports.go diff --git a/Gopkg.lock b/Gopkg.lock index 5781ff0e..a5e359ed 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -931,7 +931,7 @@ [[projects]] branch = "master" - digest = "1:bba09479e5ea003f3d14749f6e0105b6b7b6c696dc1ef76707fd138610890d63" + digest = "1:925edc3f3c4fee0b52a097616ecead6ed222d43f07d32c254cc5789c8419abe1" name = "knative.dev/pkg" packages = [ "apis", @@ -950,18 +950,18 @@ "metrics/metricskey", ] pruneopts = "T" - revision = "eec28e7bbdc9ffe4fbdc3584daa5a102d08f5cbb" + revision = "731f943e4f3d47632380d980bef53a4baeaccd29" [[projects]] branch = "master" - digest = "1:6e839a9b4183b0fbf0df609fe9a6d226c941fa9221052ba0075b25cb353588a5" + digest = "1:052dcbb5c89b1843e0f5130dc9a0e815a89079c0329bc93c86f7d8c8569b8132" name = "knative.dev/test-infra" packages = [ "scripts", "tools/dep-collector", ] pruneopts = "UT" - revision = "98a7b3bbed2dda25a17c14d1d59d24dab3b15e41" + revision = "f049c1efa7766df2044ccd02310407089a309864" [[projects]] digest = "1:8730e0150dfb2b7e173890c8b9868e7a273082ef8e39f4940e3506a481cf895c" diff --git a/vendor/knative.dev/pkg/Gopkg.lock b/vendor/knative.dev/pkg/Gopkg.lock index 800f5b47..1701cc82 100644 --- a/vendor/knative.dev/pkg/Gopkg.lock +++ b/vendor/knative.dev/pkg/Gopkg.lock @@ -618,13 +618,14 @@ [[projects]] branch = "master" - digest = "1:5578b99717f08e6480d7e0480f758749c12f9cc5da19a33a863dc7307fd699fb" + digest = "1:ab30187b3fce52d6bdedddd463ac788423992846b617ddda81d4fd1e04c3d342" name = "golang.org/x/net" packages = [ "context", "context/ctxhttp", "http/httpguts", "http2", + "http2/h2c", "http2/hpack", "idna", "internal/timeseries", @@ -1364,6 +1365,8 @@ "go.uber.org/zap/zapcore", "go.uber.org/zap/zaptest", "golang.org/x/net/context", + "golang.org/x/net/http2", + "golang.org/x/net/http2/h2c", "golang.org/x/oauth2", "golang.org/x/sync/errgroup", "google.golang.org/api/container/v1beta1", diff --git a/vendor/knative.dev/pkg/kmeta/names.go b/vendor/knative.dev/pkg/kmeta/names.go index 0e6bea26..fcbce72b 100644 --- a/vendor/knative.dev/pkg/kmeta/names.go +++ b/vendor/knative.dev/pkg/kmeta/names.go @@ -19,6 +19,7 @@ package kmeta import ( "crypto/md5" "fmt" + "strings" ) // The longest name supported by the K8s is 63. @@ -53,7 +54,9 @@ func ChildName(parent, suffix string) string { if d := longest - len(ret); d > 0 { ret += suffix[:d] } - return ret + // If due to trumming above we're terminating the string with a `-`, + // remove it. + return strings.TrimRight(ret, "-") } n = fmt.Sprintf("%s%x", parent[:head-len(suffix)], md5.Sum([]byte(parent))) } diff --git a/vendor/knative.dev/pkg/network/error_handler.go b/vendor/knative.dev/pkg/network/error_handler.go new file mode 100644 index 00000000..486518b9 --- /dev/null +++ b/vendor/knative.dev/pkg/network/error_handler.go @@ -0,0 +1,43 @@ +/* +Copyright 2019 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package network + +import ( + "io/ioutil" + "net/http" + + "go.uber.org/zap" +) + +// ErrorHandler sets up a handler suitable for use with the ErrorHandler field on +// httputil's reverse proxy. +func ErrorHandler(logger *zap.SugaredLogger) func(http.ResponseWriter, *http.Request, error) { + return func(w http.ResponseWriter, req *http.Request, err error) { + ss := readSockStat(logger) + logger.Errorw("error reverse proxying request; sockstat: "+ss, zap.Error(err)) + http.Error(w, err.Error(), http.StatusBadGateway) + } +} + +func readSockStat(logger *zap.SugaredLogger) string { + b, err := ioutil.ReadFile("/proc/net/sockstat") + if err != nil { + logger.Errorw("Unable to read sockstat", zap.Error(err)) + return "" + } + return string(b) +} diff --git a/vendor/knative.dev/pkg/network/h2c.go b/vendor/knative.dev/pkg/network/h2c.go new file mode 100644 index 00000000..683bbebe --- /dev/null +++ b/vendor/knative.dev/pkg/network/h2c.go @@ -0,0 +1,54 @@ +/* +Copyright 2019 The Knative Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package network + +import ( + "crypto/tls" + "net" + "net/http" + "time" + + "golang.org/x/net/http2" + "golang.org/x/net/http2/h2c" +) + +// NewServer returns a new HTTP Server with HTTP2 handler. +func NewServer(addr string, h http.Handler) *http.Server { + h1s := &http.Server{ + Addr: addr, + Handler: h2c.NewHandler(h, &http2.Server{}), + } + + return h1s +} + +// NewH2CTransport constructs a new H2C transport. +// That transport will reroute all HTTPS traffic to HTTP. This is +// to explicitly allow h2c (http2 without TLS) transport. +// See https://github.com/golang/go/issues/14141 for more details. +func NewH2CTransport() http.RoundTripper { + return &http2.Transport{ + AllowHTTP: true, + DialTLS: func(netw, addr string, cfg *tls.Config) (net.Conn, error) { + d := &net.Dialer{ + Timeout: DefaultConnTimeout, + KeepAlive: 5 * time.Second, + DualStack: true, + } + return d.Dial(netw, addr) + }, + } +} diff --git a/vendor/knative.dev/pkg/network/network.go b/vendor/knative.dev/pkg/network/network.go new file mode 100644 index 00000000..32015ada --- /dev/null +++ b/vendor/knative.dev/pkg/network/network.go @@ -0,0 +1,45 @@ +/* +Copyright 2019 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package network + +import ( + "time" +) + +const ( + // DefaultConnTimeout specifies a short default connection timeout + // to avoid hitting the issue fixed in + // https://github.com/kubernetes/kubernetes/pull/72534 but only + // avalailable after Kubernetes 1.14. + // + // Our connections are usually between pods in the same cluster + // like activator <-> queue-proxy, or even between containers + // within the same pod queue-proxy <-> user-container, so a + // smaller connect timeout would be justifiable. + // + // We should consider exposing this as a configuration. + DefaultConnTimeout = 200 * time.Millisecond + + // UserAgentKey is the constant for header "User-Agent". + UserAgentKey = "User-Agent" + + // ProbeHeaderName is the name of a header that can be added to + // requests to probe the knative networking layer. Requests + // with this header will not be passed to the user container or + // included in request metrics. + ProbeHeaderName = "K-Network-Probe" +) diff --git a/vendor/knative.dev/pkg/network/prober/doc.go b/vendor/knative.dev/pkg/network/prober/doc.go new file mode 100644 index 00000000..1c971e14 --- /dev/null +++ b/vendor/knative.dev/pkg/network/prober/doc.go @@ -0,0 +1,18 @@ +/* +Copyright 2019 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package prober contains functionality for implementing probing in knative serving. +package prober diff --git a/vendor/knative.dev/pkg/network/prober/prober.go b/vendor/knative.dev/pkg/network/prober/prober.go new file mode 100644 index 00000000..4030281a --- /dev/null +++ b/vendor/knative.dev/pkg/network/prober/prober.go @@ -0,0 +1,192 @@ +/* +Copyright 2019 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package prober + +import ( + "context" + "fmt" + "io/ioutil" + "net/http" + "sync" + "time" + + "go.uber.org/zap" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/wait" + "knative.dev/pkg/logging" +) + +// Preparer is a way for the caller to modify the HTTP request before it goes out. +type Preparer func(r *http.Request) *http.Request + +// Verifier is a way for the caller to validate the HTTP response after it comes back. +type Verifier func(r *http.Response, b []byte) (bool, error) + +// WithHeader sets a header in the probe request. +func WithHeader(name, value string) Preparer { + return func(r *http.Request) *http.Request { + r.Header.Set(name, value) + return r + } +} + +// WithHost sets the host in the probe request. +func WithHost(host string) Preparer { + return func(r *http.Request) *http.Request { + r.Host = host + return r + } +} + +// ExpectsBody validates that the body of the probe response matches the provided string. +func ExpectsBody(body string) Verifier { + return func(r *http.Response, b []byte) (bool, error) { + if string(b) == body { + return true, nil + } + return false, fmt.Errorf("unexpected body: want %q, got %q", body, string(b)) + } +} + +// ExpectsHeader validates that the given header of the probe response matches the provided string. +func ExpectsHeader(name, value string) Verifier { + return func(r *http.Response, _ []byte) (bool, error) { + if r.Header.Get(name) == value { + return true, nil + } + return false, fmt.Errorf("unexpected header %q: want %q, got %q", name, value, r.Header.Get(name)) + } +} + +// ExpectsStatusCodes validates that the given status code of the probe response matches the provided int. +func ExpectsStatusCodes(statusCodes []int) Verifier { + return func(r *http.Response, _ []byte) (bool, error) { + for _, v := range statusCodes { + if r.StatusCode == v { + return true, nil + } + } + return false, fmt.Errorf("unexpected status code: want %v, got %v", statusCodes, r.StatusCode) + } +} + +// Do sends a single probe to given target, e.g. `http://revision.default.svc.cluster.local:81`. +// Do returns whether the probe was successful or not, or there was an error probing. +func Do(ctx context.Context, transport http.RoundTripper, target string, ops ...interface{}) (bool, error) { + req, err := http.NewRequest(http.MethodGet, target, nil) + if err != nil { + return false, fmt.Errorf("%s is not a valid URL: %v", target, err) + } + for _, op := range ops { + if po, ok := op.(Preparer); ok { + req = po(req) + } + } + + req = req.WithContext(ctx) + resp, err := transport.RoundTrip(req) + if err != nil { + return false, fmt.Errorf("error roundtripping %s: %v", target, err) + } + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return false, fmt.Errorf("error reading body: %v", err) + } + + for _, op := range ops { + if vo, ok := op.(Verifier); ok { + if ok, err := vo(resp, body); err != nil || !ok { + return false, err + } + } + } + return true, nil +} + +// Done is a callback that is executed when the async probe has finished. +// `arg` is given by the caller at the offering time, while `success` and `err` +// are the return values of the `Do` call. +// It is assumed that the opaque arg is consistent for a given target and +// we will coalesce concurrent Offer invocations on target. +type Done func(arg interface{}, success bool, err error) + +// Manager manages async probes and makes sure we run concurrently only a single +// probe for the same key. +type Manager struct { + cb Done + // NB: it is paramount to use a transport that will close the connection + // after every request here. Otherwise the cached connections will prohibit + // scaling to zero, due to unsuccessful probes to the Activator. + transport http.RoundTripper + + // mu guards keys. + mu sync.Mutex + keys sets.String +} + +// New creates a new Manager, that will invoke the given callback when +// async probing is finished. +func New(cb Done, transport http.RoundTripper) *Manager { + return &Manager{ + keys: sets.NewString(), + cb: cb, + transport: transport, + } +} + +// Offer executes asynchronous probe using `target` as the key. +// If a probe with the same key already exists, Offer will return false and the +// call is discarded. If the request is accepted, Offer returns true. +// Otherwise Offer starts a goroutine that periodically executes +// `Do`, until timeout is reached, the probe succeeds, or fails with an error. +// In the end the callback is invoked with the provided `arg` and probing results. +func (m *Manager) Offer(ctx context.Context, target string, arg interface{}, period, timeout time.Duration, ops ...interface{}) bool { + m.mu.Lock() + defer m.mu.Unlock() + if m.keys.Has(target) { + return false + } + m.keys.Insert(target) + m.doAsync(ctx, target, arg, period, timeout, ops...) + return true +} + +// doAsync starts a go routine that probes the target with given period. +func (m *Manager) doAsync(ctx context.Context, target string, arg interface{}, period, timeout time.Duration, ops ...interface{}) { + logger := logging.FromContext(ctx) + go func() { + defer func() { + m.mu.Lock() + defer m.mu.Unlock() + m.keys.Delete(target) + }() + var ( + result bool + inErr error + ) + err := wait.PollImmediate(period, timeout, func() (bool, error) { + result, inErr = Do(ctx, m.transport, target, ops...) + // Do not return error, which is from verifierError, as retry is expected until timeout. + return result, nil + }) + if inErr != nil { + logger.Errorw("Unable to read sockstat", zap.Error(inErr)) + } + m.cb(arg, result, err) + }() +} diff --git a/vendor/knative.dev/pkg/network/transports.go b/vendor/knative.dev/pkg/network/transports.go new file mode 100644 index 00000000..35747fa3 --- /dev/null +++ b/vendor/knative.dev/pkg/network/transports.go @@ -0,0 +1,120 @@ +/* +Copyright 2019 The Knative Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package network + +import ( + "context" + "errors" + "net" + "net/http" + "time" + + "k8s.io/apimachinery/pkg/util/wait" +) + +// RoundTripperFunc implementation roundtrips a request. +type RoundTripperFunc func(*http.Request) (*http.Response, error) + +// RoundTrip implements http.RoundTripper. +func (rt RoundTripperFunc) RoundTrip(r *http.Request) (*http.Response, error) { + return rt(r) +} + +func newAutoTransport(v1 http.RoundTripper, v2 http.RoundTripper) http.RoundTripper { + return RoundTripperFunc(func(r *http.Request) (*http.Response, error) { + t := v1 + if r.ProtoMajor == 2 { + t = v2 + } + return t.RoundTrip(r) + }) +} + +const sleepTO = 30 * time.Millisecond + +var backOffTemplate = wait.Backoff{ + Duration: 50 * time.Millisecond, + Factor: 1.4, + Jitter: 0.1, // At most 10% jitter. + Steps: 15, +} + +var errDialTimeout = errors.New("timed out dialing") + +// dialWithBackOff executes `net.Dialer.DialContext()` with exponentially increasing +// dial timeouts. In addition it sleeps with random jitter between tries. +func dialWithBackOff(ctx context.Context, network, address string) (net.Conn, error) { + return dialBackOffHelper(ctx, network, address, backOffTemplate, sleepTO) +} + +func dialBackOffHelper(ctx context.Context, network, address string, bo wait.Backoff, sleep time.Duration) (net.Conn, error) { + dialer := &net.Dialer{ + Timeout: bo.Duration, // Initial duration. + KeepAlive: 5 * time.Second, + DualStack: true, + } + for { + c, err := dialer.DialContext(ctx, network, address) + if err != nil { + if err, ok := err.(net.Error); ok && err.Timeout() { + if bo.Steps < 1 { + break + } + dialer.Timeout = bo.Step() + time.Sleep(wait.Jitter(sleep, 1.0)) // Sleep with jitter. + continue + } + return nil, err + } + return c, nil + } + return nil, errDialTimeout +} + +func newHTTPTransport(connTimeout time.Duration, disableKeepAlives bool) http.RoundTripper { + return &http.Transport{ + // Those match net/http/transport.go + Proxy: http.ProxyFromEnvironment, + MaxIdleConns: 1000, + MaxIdleConnsPerHost: 100, + IdleConnTimeout: 5 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + DisableKeepAlives: disableKeepAlives, + + // This is bespoke. + DialContext: dialWithBackOff, + } +} + +// NewProberTransport creates a RoundTripper that is useful for probing, +// since it will not cache connections. +func NewProberTransport() http.RoundTripper { + return newAutoTransport( + newHTTPTransport(DefaultConnTimeout, true /*disable keep-alives*/), + NewH2CTransport()) +} + +// NewAutoTransport creates a RoundTripper that can use appropriate transport +// based on the request's HTTP version. +func NewAutoTransport() http.RoundTripper { + return newAutoTransport( + newHTTPTransport(DefaultConnTimeout, false /*disable keep-alives*/), + NewH2CTransport()) +} + +// AutoTransport uses h2c for HTTP2 requests and falls back to `http.DefaultTransport` for all others +var AutoTransport = NewAutoTransport() diff --git a/vendor/knative.dev/test-infra/scripts/library.sh b/vendor/knative.dev/test-infra/scripts/library.sh index ca0d6d47..d686114c 100755 --- a/vendor/knative.dev/test-infra/scripts/library.sh +++ b/vendor/knative.dev/test-infra/scripts/library.sh @@ -443,6 +443,29 @@ function start_latest_knative_serving() { start_knative_serving "${KNATIVE_SERVING_RELEASE}" } +# Install Knative Eventing in the current cluster. +# Parameters: $1 - Knative Eventing manifest. +function start_knative_eventing() { + header "Starting Knative Eventing" + subheader "Installing Knative Eventing" + echo "Installing Eventing CRDs from $1" + kubectl apply --selector knative.dev/crd-install=true -f "$1" + echo "Installing the rest of eventing components from $1" + kubectl apply -f "$1" + wait_until_pods_running knative-eventing || return 1 +} + +# Install the stable release Knative/eventing in the current cluster. +# Parameters: $1 - Knative Eventing version number, e.g. 0.6.0. +function start_release_knative_eventing() { + start_knative_eventing "https://storage.googleapis.com/knative-releases/eventing/previous/v$1/release.yaml" +} + +# Install the latest stable Knative Eventing in the current cluster. +function start_latest_knative_eventing() { + start_knative_eventing "${KNATIVE_EVENTING_RELEASE}" +} + # Run a go tool, installing it first if necessary. # Parameters: $1 - tool package/dir for go get/install. # $2 - tool to run.