controller: Increase HTTP ReadHeaderTimeout to 15s (#9272)

04a66ba added a `ReadHeaderTimeout` to our HTTP servers (at gosec's
insistence). We chose a fairly arbitrary timeout of 10s. This
configuration causes any connection that has been idle for 10s to be
torn down by the server. Unfortunately, this timeout value matches the
default Kubernetes probe interval and the default linkerd-viz scrape
interval. This can cause probes to race the timeout so that the
connection is healthy from the proxy's point of view and a request is
sent on the connection exactly as the server drops the connection.
These request failures cause controller success rate to appear degraded.

To correct this, this change raises the timeout to 15s so that the
timeout no longer matches the default probe interval.

The proxy's HTTP client is supposed to [retry] requests that encounter
this type of error. We should follow up by doing more research into why
that is not occurring in this situation.

[retry]: https://docs.rs/hyper/0.14.20/hyper/client/struct.Builder.html#method.retry_canceled_requests

Signed-off-by: Oliver Gould <ver@buoyant.io>

controller/webhook/server.go

@@ -63,7 +63,7 @@ func NewServer(
 
 	server := &http.Server{
 		Addr:              addr,
-		ReadHeaderTimeout: 10 * time.Second,
+		ReadHeaderTimeout: 15 * time.Second,
 		TLSConfig: &tls.Config{
 			MinVersion: tls.VersionTLS12,
 		},

controller/webhook/server_test.go

@@ -16,7 +16,7 @@ import (
 
 var mockHTTPServer = &http.Server{
 	Addr:              ":0",
-	ReadHeaderTimeout: 10 * time.Second,
+	ReadHeaderTimeout: 15 * time.Second,
 	TLSConfig: &tls.Config{
 		MinVersion: tls.VersionTLS12,
 	},

pkg/admin/admin.go

@@ -25,7 +25,7 @@ func NewServer(addr string, enablePprof bool) *http.Server {
 	return &http.Server{
 		Addr:              addr,
 		Handler:           h,
-		ReadHeaderTimeout: 10 * time.Second,
+		ReadHeaderTimeout: 15 * time.Second,
 	}
 }
 

viz/tap/api/server.go

@@ -71,7 +71,7 @@ func NewServer(
 
 	httpServer := &http.Server{
 		Addr:              addr,
-		ReadHeaderTimeout: 10 * time.Second,
+		ReadHeaderTimeout: 15 * time.Second,
 		TLSConfig: &tls.Config{
 			ClientAuth: tls.VerifyClientCertIfGiven,
 			ClientCAs:  clientCertPool,

web/srv/server.go

@@ -21,7 +21,7 @@ import (
 )
 
 const (
-	timeout = 10 * time.Second
+	timeout = 15 * time.Second
 
 	// statExpiration indicates when items in the stat cache expire.
 	statExpiration = 1500 * time.Millisecond