Graceful shutdown for admin server (#6817)

* Graceful shutdown for admin server

Signed-off-by: Stepan Rabotkin <epicstyt@gmail.com>

controller/cmd/destination/main.go

@@ -131,11 +131,17 @@ func Main(args []string) {
 		server.Serve(lis)
 	}()
 
-	go admin.StartServer(*metricsAddr)
+	adminServer := admin.NewServer(*metricsAddr)
+
+	go func() {
+		log.Infof("starting admin server on %s", *metricsAddr)
+		adminServer.ListenAndServe()
+	}()
 
 	<-stop
 
 	log.Infof("shutting down gRPC server on %s", *addr)
 	close(done)
 	server.GracefulStop()
+	adminServer.Shutdown(ctx)
 }

controller/cmd/identity/main.go

@@ -167,7 +167,13 @@ func Main(args []string) {
 	//
 	// Bind and serve
 	//
-	go admin.StartServer(*adminAddr)
+	adminServer := admin.NewServer(*adminAddr)
+
+	go func() {
+		log.Infof("starting admin server on %s", *adminAddr)
+		adminServer.ListenAndServe()
+	}()
+
 	lis, err := net.Listen("tcp", *addr)
 	if err != nil {
 		log.Fatalf("Failed to listen on %s: %s", *addr, err)
@@ -187,4 +193,5 @@ func Main(args []string) {
 	<-stop
 	log.Infof("shutting down gRPC server on %s", *addr)
 	srv.GracefulStop()
+	adminServer.Shutdown(ctx)
 }

controller/webhook/launcher.go

@@ -40,7 +40,13 @@ func Launch(
 	go s.Start()
 
 	k8sAPI.Sync(nil)
-	go admin.StartServer(metricsAddr)
+
+	adminServer := admin.NewServer(metricsAddr)
+
+	go func() {
+		log.Infof("starting admin server on %s", metricsAddr)
+		adminServer.ListenAndServe()
+	}()
 
 	<-stop
 	log.Info("shutting down webhook server")
@@ -49,4 +55,6 @@ func Launch(
 	if err := s.Shutdown(ctx); err != nil {
 		log.Error(err)
 	}
+
+	adminServer.Shutdown(ctx)
 }

multicluster/cmd/service-mirror/main.go

@@ -75,7 +75,13 @@ func Main(args []string) {
 	linkClient := k8sAPI.DynamicClient.Resource(multicluster.LinkGVR).Namespace(*namespace)
 
 	metrics := servicemirror.NewProbeMetricVecs()
-	go admin.StartServer(*metricsAddr)
+
+	adminServer := admin.NewServer(*metricsAddr)
+
+	go func() {
+		log.Infof("starting admin server on %s", *metricsAddr)
+		adminServer.ListenAndServe()
+	}()
 
 	controllerK8sAPI.Sync(nil)
 

pkg/admin/admin.go

@@ -7,22 +7,22 @@ import (
 	"strings"
 
 	"github.com/prometheus/client_golang/prometheus/promhttp"
-	log "github.com/sirupsen/logrus"
 )
 
 type handler struct {
 	promHandler http.Handler
 }
 
-// StartServer starts an admin server listening on a given address.
+// NewServer returns an initialized `http.Server`, configured to listen on an address.
-func StartServer(addr string) {
+func NewServer(addr string) *http.Server {
-	log.Infof("starting admin server on %s", addr)
-
 	h := &handler{
 		promHandler: promhttp.Handler(),
 	}
 
-	log.Fatal(http.ListenAndServe(addr, h))
+	return &http.Server{
+		Addr:    addr,
+		Handler: h,
+	}
 }
 
 func (h *handler) ServeHTTP(w http.ResponseWriter, req *http.Request) {

viz/metrics-api/cmd/main.go

@@ -79,10 +79,16 @@ func main() {
 		server.ListenAndServe()
 	}()
 
-	go admin.StartServer(*metricsAddr)
+	adminServer := admin.NewServer(*metricsAddr)
+
+	go func() {
+		log.Infof("starting admin server on %s", *metricsAddr)
+		adminServer.ListenAndServe()
+	}()
 
 	<-stop
 
 	log.Infof("shutting down HTTP server on %+v", *addr)
 	server.Shutdown(ctx)
+	adminServer.Shutdown(ctx)
 }

viz/tap/api/main.go

@@ -63,8 +63,16 @@ func Main(args []string) {
 	}
 	k8sAPI.Sync(nil)
 	go apiServer.Start(ctx)
-	go admin.StartServer(*metricsAddr)
+
+	adminServer := admin.NewServer(*metricsAddr)
+
+	go func() {
+		log.Infof("starting admin server on %s", *metricsAddr)
+		adminServer.ListenAndServe()
+	}()
+
 	<-stop
 	log.Infof("shutting down APIServer on %s", *apiServerAddr)
 	apiServer.Shutdown(ctx)
+	adminServer.Shutdown(ctx)
 }

web/main.go

@@ -100,7 +100,12 @@ func main() {
 		server.ListenAndServe()
 	}()
 
-	go admin.StartServer(*metricsAddr)
+	adminServer := admin.NewServer(*metricsAddr)
+
+	go func() {
+		log.Infof("starting admin server on %s", *metricsAddr)
+		adminServer.ListenAndServe()
+	}()
 
 	<-stop
 
@@ -108,6 +113,7 @@ func main() {
 	ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
 	defer cancel()
 	server.Shutdown(ctx)
+	adminServer.Shutdown(ctx)
 }
 
 func getUUIDAndVersion(ctx context.Context, k8sAPI *k8s.KubernetesAPI, controllerNamespace string) (string, string) {