From aa5d4f9f320387a9c3cf35b5b7d8cc3398069ae4 Mon Sep 17 00:00:00 2001 From: Mike Spreitzer Date: Mon, 19 Feb 2018 14:18:07 -0500 Subject: [PATCH] Fixes for HTTP/2 max streams per connection setting This PR makes two changes. One is to introduce a parameter for the HTTP/2 setting that an api-server sends to its clients telling them how many streams they may have concurrently open in an HTTP/2 connection. If left at its default value of zero, this means to use the default in golang's HTTP/2 code (which is currently 250). The other change is to make the recommended options for an aggregated api-server set this limit to 1000. The limit of 250 is annoyingly low for the use case of many controllers watching objects of Kinds served by an aggregated api-server reached through the main api-server (in its mode as a proxy for the aggregated api-server, in which it uses a single HTTP/2 connection for all calls proxied to that aggregated api-server). Fixes #60042 Kubernetes-commit: 201c11f147c85b029665915bee3a62eea19d6d57 --- pkg/server/BUILD | 1 + pkg/server/config.go | 4 ++++ pkg/server/options/recommended.go | 10 +++++++++- pkg/server/options/serving.go | 12 +++++++++++- pkg/server/serve.go | 7 +++++++ 5 files changed, 32 insertions(+), 2 deletions(-) diff --git a/pkg/server/BUILD b/pkg/server/BUILD index 1603aa015..c1073fc1d 100644 --- a/pkg/server/BUILD +++ b/pkg/server/BUILD @@ -97,6 +97,7 @@ go_library( "//vendor/github.com/go-openapi/spec:go_default_library", "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/pborman/uuid:go_default_library", + "//vendor/golang.org/x/net/http2:go_default_library", "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apimachinery:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apimachinery/registered:go_default_library", diff --git a/pkg/server/config.go b/pkg/server/config.go index a950f80f8..2f4c48b45 100644 --- a/pkg/server/config.go +++ b/pkg/server/config.go @@ -227,6 +227,10 @@ type SecureServingInfo struct { // CipherSuites optionally overrides the list of allowed cipher suites for the server. // Values are from tls package constants (https://golang.org/pkg/crypto/tls/#pkg-constants). CipherSuites []uint16 + + // HTTP2MaxStreamsPerConnection is the limit that the api server imposes on each client. + // A value of zero means to use the default provided by golang's HTTP/2 support. + HTTP2MaxStreamsPerConnection int } type AuthenticationInfo struct { diff --git a/pkg/server/options/recommended.go b/pkg/server/options/recommended.go index c2aa6d576..ebd750c2b 100644 --- a/pkg/server/options/recommended.go +++ b/pkg/server/options/recommended.go @@ -44,9 +44,17 @@ type RecommendedOptions struct { } func NewRecommendedOptions(prefix string, codec runtime.Codec) *RecommendedOptions { + sso := NewSecureServingOptions() + + // We are composing recommended options for an aggregated api-server, + // whose client is typically a proxy multiplexing many operations --- + // notably including long-running ones --- into one HTTP/2 connection + // into this server. So allow many concurrent operations. + sso.HTTP2MaxStreamsPerConnection = 1000 + return &RecommendedOptions{ Etcd: NewEtcdOptions(storagebackend.NewDefaultConfig(prefix, codec)), - SecureServing: WithLoopback(NewSecureServingOptions()), + SecureServing: WithLoopback(sso), Authentication: NewDelegatingAuthenticationOptions(), Authorization: NewDelegatingAuthorizationOptions(), Audit: NewAuditOptions(), diff --git a/pkg/server/options/serving.go b/pkg/server/options/serving.go index 4d4a8f518..c4c2bf986 100644 --- a/pkg/server/options/serving.go +++ b/pkg/server/options/serving.go @@ -55,6 +55,10 @@ type SecureServingOptions struct { // MinTLSVersion is the minimum TLS version supported. // Values are from tls package constants (https://golang.org/pkg/crypto/tls/#pkg-constants). MinTLSVersion string + + // HTTP2MaxStreamsPerConnection is the limit that the api server imposes on each client. + // A value of zero means to use the default provided by golang's HTTP/2 support. + HTTP2MaxStreamsPerConnection int } type CertKey struct { @@ -148,6 +152,11 @@ func (s *SecureServingOptions) AddFlags(fs *pflag.FlagSet) { "--tls-sni-cert-key multiple times. "+ "Examples: \"example.crt,example.key\" or \"foo.crt,foo.key:*.foo.com,foo.com\".") + fs.IntVar(&s.HTTP2MaxStreamsPerConnection, "http2-max-streams-per-connection", s.HTTP2MaxStreamsPerConnection, ""+ + "The limit that the server gives to clients for "+ + "the maximum number of streams in an HTTP/2 connection. "+ + "Zero means to use golang's default.") + // TODO remove this flag in 1.11. The flag had no effect before this will prevent scripts from immediately failing on upgrade. fs.String("tls-ca-file", "", "This flag has no effect.") fs.MarkDeprecated("tls-ca-file", "This flag has no effect.") @@ -179,7 +188,8 @@ func (s *SecureServingOptions) ApplyTo(config **server.SecureServingInfo) error } *config = &server.SecureServingInfo{ - Listener: s.Listener, + Listener: s.Listener, + HTTP2MaxStreamsPerConnection: s.HTTP2MaxStreamsPerConnection, } c := *config diff --git a/pkg/server/serve.go b/pkg/server/serve.go index 71a3a3479..7cfc10382 100644 --- a/pkg/server/serve.go +++ b/pkg/server/serve.go @@ -27,6 +27,7 @@ import ( "time" "github.com/golang/glog" + "golang.org/x/net/http2" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/validation" @@ -86,6 +87,12 @@ func (s *SecureServingInfo) Serve(handler http.Handler, shutdownTimeout time.Dur secureServer.TLSConfig.ClientCAs = s.ClientCA } + if s.HTTP2MaxStreamsPerConnection > 0 { + http2.ConfigureServer(secureServer, &http2.Server{ + MaxConcurrentStreams: uint32(s.HTTP2MaxStreamsPerConnection), + }) + } + glog.Infof("Serving securely on %s", secureServer.Addr) return RunServer(secureServer, s.Listener, shutdownTimeout, stopCh) }