pass listener to genericapiserver

Kubernetes-commit: 6ba30f678c232793430a98770e7a851f1e814fd2

pkg/server/BUILD

@@ -71,7 +71,6 @@ go_library(
         "//vendor/github.com/go-openapi/spec:go_default_library",
         "//vendor/github.com/golang/glog:go_default_library",
         "//vendor/github.com/pborman/uuid:go_default_library",
-        "//vendor/github.com/pkg/errors:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apimachinery:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apimachinery/registered:go_default_library",

pkg/server/config.go

@@ -203,11 +203,8 @@ type RecommendedConfig struct {
 }
 
 type SecureServingInfo struct {
-	// BindAddress is the ip:port to serve on
+	// Listener is the secure server network listener.
-	BindAddress string
+	Listener net.Listener
-	// BindNetwork is the type of network to bind to - defaults to "tcp", accepts "tcp",
-	// "tcp4", and "tcp6".
-	BindNetwork string
 
 	// Cert is the main server cert which is used if SNI does not match. Cert must be non-nil and is
 	// allowed to be in SNICerts.

pkg/server/config_selfclient.go

@@ -32,7 +32,7 @@ func (s *SecureServingInfo) NewLoopbackClientConfig(token string, loopbackCert [
 		return nil, nil
 	}
 
-	host, port, err := LoopbackHostPort(s.BindAddress)
+	host, port, err := LoopbackHostPort(s.Listener.Addr().String())
 	if err != nil {
 		return nil, err
 	}
@@ -64,7 +64,7 @@ func LoopbackHostPort(bindAddress string) (string, string, error) {
 	}
 
 	// Value is expected to be an IP or DNS name, not "0.0.0.0".
-	if host == "0.0.0.0" {
+	if host == "0.0.0.0" || host == "::" {
 		host = "localhost"
 		// Get ip of local interface, but fall back to "localhost".
 		// Note that "localhost" is resolved with the external nameserver first with Go's stdlib.

pkg/server/config_selfclient_test.go

@@ -43,4 +43,26 @@ func TestLoopbackHostPort(t *testing.T) {
 	if port != "443" {
 		t.Fatalf("expected 443 as port, got %q", port)
 	}
+
+	host, port, err = LoopbackHostPort("[ff06:0:0:0:0:0:0:c3]:443")
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if host != "ff06:0:0:0:0:0:0:c3" {
+		t.Fatalf("expected ff06:0:0:0:0:0:0:c3 as host, got %q", host)
+	}
+	if port != "443" {
+		t.Fatalf("expected 443 as port, got %q", port)
+	}
+
+	host, port, err = LoopbackHostPort("[::]:443")
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if ip := net.ParseIP(host); ip == nil || !ip.IsLoopback() {
+		t.Fatalf("expected host to be loopback, got %q", host)
+	}
+	if port != "443" {
+		t.Fatalf("expected 443 as port, got %q", port)
+	}
 }

pkg/server/genericapiserver.go

@@ -100,9 +100,6 @@ type GenericAPIServer struct {
 
 	SecureServingInfo *SecureServingInfo
 
-	// numerical ports, set after listening
-	effectiveSecurePort int
-
 	// ExternalAddress is the address (hostname or IP and port) that should be used in
 	// external (public internet) URLs for this GenericAPIServer.
 	ExternalAddress string
@@ -338,11 +335,6 @@ func (s preparedGenericAPIServer) NonBlockingRun(stopCh <-chan struct{}) error {
 	return nil
 }
 
-// EffectiveSecurePort returns the secure port we bound to.
-func (s *GenericAPIServer) EffectiveSecurePort() int {
-	return s.effectiveSecurePort
-}
-
 // installAPIResources is a private method for installing the REST storage backing each api groupversionresource
 func (s *GenericAPIServer) installAPIResources(apiPrefix string, apiGroupInfo *APIGroupInfo) error {
 	for _, groupVersion := range apiGroupInfo.GroupMeta.GroupVersions {

pkg/server/genericapiserver_test.go

@@ -549,7 +549,15 @@ func TestGracefulShutdown(t *testing.T) {
 		Handler: s.Handler,
 	}
 	stopCh := make(chan struct{})
-	serverPort, err := RunServer(insecureServer, "tcp", 10*time.Second, stopCh)
+
+	ln, err := net.Listen("tcp", insecureServer.Addr)
+	if err != nil {
+		t.Errorf("failed to listen on %v: %v", insecureServer.Addr, err)
+	}
+
+	// get port
+	serverPort := ln.Addr().(*net.TCPAddr).Port
+	err = RunServer(insecureServer, ln, 10*time.Second, stopCh)
 	if err != nil {
 		t.Errorf("RunServer err: %v", err)
 	}

pkg/server/options/serving.go

@@ -38,6 +38,14 @@ import (
 type SecureServingOptions struct {
 	BindAddress net.IP
 	BindPort    int
+	// BindNetwork is the type of network to bind to - defaults to "tcp", accepts "tcp",
+	// "tcp4", and "tcp6".
+	BindNetwork string
+
+	// Listener is the secure server network listener.
+	// either Listener or BindAddress/BindPort/BindNetwork is set,
+	// if Listener is set, use it and omit BindAddress/BindPort/BindNetwork.
+	Listener net.Listener
 
 	// ServerCert is the TLS cert info for serving secure traffic
 	ServerCert GeneratableKeyCert
@@ -147,14 +155,25 @@ func (s *SecureServingOptions) ApplyTo(c *server.Config) error {
 	if s == nil {
 		return nil
 	}
-
 	if s.BindPort <= 0 {
 		return nil
 	}
+
+	if s.Listener == nil {
+		var err error
+		addr := net.JoinHostPort(s.BindAddress.String(), strconv.Itoa(s.BindPort))
+		s.Listener, s.BindPort, err = CreateListener(s.BindNetwork, addr)
+		if err != nil {
+			return fmt.Errorf("failed to create listener: %v", err)
+		}
+	}
+
 	if err := s.applyServingInfoTo(c); err != nil {
 		return err
 	}
 
+	c.SecureServingInfo.Listener = s.Listener
+
 	// create self-signed cert+key with the fake server.LoopbackClientServerNameOverride and
 	// let the server return it when the loopback client connects.
 	certPem, keyPem, err := certutil.GenerateSelfSignedCertKey(server.LoopbackClientServerNameOverride, nil, nil)
@@ -184,16 +203,9 @@ func (s *SecureServingOptions) ApplyTo(c *server.Config) error {
 }
 
 func (s *SecureServingOptions) applyServingInfoTo(c *server.Config) error {
-	if s.BindPort <= 0 {
+	secureServingInfo := &server.SecureServingInfo{}
-		return nil
-	}
-
-	secureServingInfo := &server.SecureServingInfo{
-		BindAddress: net.JoinHostPort(s.BindAddress.String(), strconv.Itoa(s.BindPort)),
-	}
 
 	serverCertFile, serverKeyFile := s.ServerCert.CertKey.CertFile, s.ServerCert.CertKey.KeyFile
-
 	// load main cert
 	if len(serverCertFile) != 0 || len(serverKeyFile) != 0 {
 		tlsCert, err := tls.LoadX509KeyPair(serverCertFile, serverKeyFile)
@@ -250,7 +262,7 @@ func (s *SecureServingOptions) MaybeDefaultWithSelfSignedCerts(publicAddress str
 		return nil
 	}
 	keyCert := &s.ServerCert.CertKey
-	if s.BindPort == 0 || len(keyCert.CertFile) != 0 || len(keyCert.KeyFile) != 0 {
+	if len(keyCert.CertFile) != 0 || len(keyCert.KeyFile) != 0 {
 		return nil
 	}
 
@@ -286,3 +298,22 @@ func (s *SecureServingOptions) MaybeDefaultWithSelfSignedCerts(publicAddress str
 
 	return nil
 }
+
+func CreateListener(network, addr string) (net.Listener, int, error) {
+	if len(network) == 0 {
+		network = "tcp"
+	}
+	ln, err := net.Listen(network, addr)
+	if err != nil {
+		return nil, 0, fmt.Errorf("failed to listen on %v: %v", addr, err)
+	}
+
+	// get port
+	tcpAddr, ok := ln.Addr().(*net.TCPAddr)
+	if !ok {
+		ln.Close()
+		return nil, 0, fmt.Errorf("invalid listen address: %q", ln.Addr().String())
+	}
+
+	return ln, tcpAddr.Port, nil
+}

pkg/server/options/serving_test.go

@@ -47,6 +47,7 @@ import (
 	utilflag "k8s.io/apiserver/pkg/util/flag"
 	"k8s.io/client-go/discovery"
 	restclient "k8s.io/client-go/rest"
+	"strconv"
 )
 
 func setUp(t *testing.T) Config {
@@ -481,6 +482,15 @@ NextTest:
 				},
 				SNICertKeys: namedCertKeys,
 			}
+			// use a random free port
+			ln, err := net.Listen("tcp", "127.0.0.1:0")
+			if err != nil {
+				t.Errorf("failed to listen on 127.0.0.1:0")
+			}
+
+			secureOptions.Listener = ln
+			// get port
+			secureOptions.BindPort = ln.Addr().(*net.TCPAddr).Port
 			config.LoopbackClientConfig = &restclient.Config{}
 			if err := secureOptions.ApplyTo(&config); err != nil {
 				t.Errorf("%q - failed applying the SecureServingOptions: %v", title, err)
@@ -493,9 +503,6 @@ NextTest:
 				return
 			}
 
-			// patch in a 0-port to enable auto port allocation
-			s.SecureServingInfo.BindAddress = "127.0.0.1:0"
-
 			// add poststart hook to know when the server is up.
 			startedCh := make(chan struct{})
 			s.AddPostStartHook("test-notifier", func(context PostStartHookContext) error {
@@ -517,9 +524,8 @@ NextTest:
 
 			<-startedCh
 
-			effectiveSecurePort := fmt.Sprintf("%d", preparedServer.EffectiveSecurePort())
 			// try to dial
-			addr := fmt.Sprintf("localhost:%s", effectiveSecurePort)
+			addr := fmt.Sprintf("localhost:%d", secureOptions.BindPort)
 			t.Logf("Dialing %s as %q", addr, test.ServerName)
 			conn, err := tls.Dial("tcp", addr, &tls.Config{
 				RootCAs:    roots,
@@ -547,7 +553,7 @@ NextTest:
 			if len(test.LoopbackClientBindAddressOverride) != 0 {
 				host = test.LoopbackClientBindAddressOverride
 			}
-			s.LoopbackClientConfig.Host = net.JoinHostPort(host, effectiveSecurePort)
+			s.LoopbackClientConfig.Host = net.JoinHostPort(host, strconv.Itoa(secureOptions.BindPort))
 			if test.ExpectLoopbackClientError {
 				if err == nil {
 					t.Errorf("%q - expected error creating loopback client config", title)

pkg/server/serve.go

@@ -26,11 +26,10 @@ import (
 	"strings"
 	"time"
 
+	"github.com/golang/glog"
+
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apimachinery/pkg/util/validation"
-
-	"github.com/golang/glog"
-	"github.com/pkg/errors"
 )
 
 const (
@@ -41,8 +40,12 @@ const (
 // be loaded or the initial listen call fails. The actual server loop (stoppable by closing
 // stopCh) runs in a go routine, i.e. serveSecurely does not block.
 func (s *GenericAPIServer) serveSecurely(stopCh <-chan struct{}) error {
+	if s.SecureServingInfo.Listener == nil {
+		return fmt.Errorf("listener must not be nil")
+	}
+
 	secureServer := &http.Server{
-		Addr:           s.SecureServingInfo.BindAddress,
+		Addr:           s.SecureServingInfo.Listener.Addr().String(),
 		Handler:        s.Handler,
 		MaxHeaderBytes: 1 << 20,
 		TLSConfig: &tls.Config{
@@ -83,33 +86,22 @@ func (s *GenericAPIServer) serveSecurely(stopCh <-chan struct{}) error {
 		secureServer.TLSConfig.ClientCAs = s.SecureServingInfo.ClientCA
 	}
 
-	glog.Infof("Serving securely on %s", s.SecureServingInfo.BindAddress)
+	glog.Infof("Serving securely on %s", secureServer.Addr)
-	var err error
+	err := RunServer(secureServer, s.SecureServingInfo.Listener, s.ShutdownTimeout, stopCh)
-	s.effectiveSecurePort, err = RunServer(secureServer, s.SecureServingInfo.BindNetwork, s.ShutdownTimeout, stopCh)
 	return err
 }
 
-// RunServer listens on the given port, then spawns a go-routine continuously serving
+// RunServer listens on the given port if listener is not given,
-// until the stopCh is closed. The port is returned. This function does not block.
+// then spawns a go-routine continuously serving
-func RunServer(server *http.Server, network string, shutDownTimeout time.Duration, stopCh <-chan struct{}) (int, error) {
+// until the stopCh is closed. This function does not block.
-	if len(server.Addr) == 0 {
+func RunServer(
-		return 0, errors.New("address cannot be empty")
+	server *http.Server,
-	}
+	ln net.Listener,
-
+	shutDownTimeout time.Duration,
-	if len(network) == 0 {
+	stopCh <-chan struct{},
-		network = "tcp"
+) error {
-	}
+	if ln == nil {
-
+		return fmt.Errorf("listener must not be nil")
-	ln, err := net.Listen(network, server.Addr)
-	if err != nil {
-		return 0, fmt.Errorf("failed to listen on %v: %v", server.Addr, err)
-	}
-
-	// get port
-	tcpAddr, ok := ln.Addr().(*net.TCPAddr)
-	if !ok {
-		ln.Close()
-		return 0, fmt.Errorf("invalid listen address: %q", ln.Addr().String())
 	}
 
 	// Shutdown server gracefully.
@@ -131,7 +123,7 @@ func RunServer(server *http.Server, network string, shutDownTimeout time.Duratio
 
 		err := server.Serve(listener)
 
-		msg := fmt.Sprintf("Stopped listening on %s", tcpAddr.String())
+		msg := fmt.Sprintf("Stopped listening on %s", ln.Addr().String())
 		select {
 		case <-stopCh:
 			glog.Info(msg)
@@ -140,7 +132,7 @@ func RunServer(server *http.Server, network string, shutDownTimeout time.Duratio
 		}
 	}()
 
-	return tcpAddr.Port, nil
+	return nil
 }
 
 type NamedTLSCert struct {