Merge pull request #5437 from liangyuanpeng/scheduler_options

Standardize the health probe and metrics arguments of scheduler.

artifacts/deploy/karmada-scheduler.yaml

@@ -36,7 +36,8 @@ spec:
             - /bin/karmada-scheduler
             - --kubeconfig=/etc/kubeconfig
             - --bind-address=0.0.0.0
-            - --secure-port=10351
+            - --metrics-bind-address=0.0.0.0:10351
+            - --health-probe-bind-address=0.0.0.0:10351
             - --enable-scheduler-estimator=true
             - --scheduler-estimator-ca-file=/etc/karmada/pki/ca.crt
             - --scheduler-estimator-cert-file=/etc/karmada/pki/karmada.crt

cmd/scheduler/app/options/options.go

@@ -52,10 +52,24 @@ type Options struct {
 	KubeConfig     string
 	Master         string
 	// BindAddress is the IP address on which to listen for the --secure-port port.
+	// Deprecated: To specify the TCP address for serving health probes, use HealthProbeBindAddress instead. To specify the TCP address for serving prometheus metrics, use MetricsBindAddress instead. This will be removed in release 1.12+.
 	BindAddress string
 	// SecurePort is the port that the server serves at.
+	// Deprecated: To specify the TCP address for serving health probes, use HealthProbeBindAddress instead. To specify the TCP address for serving prometheus metrics, use MetricsBindAddress instead. This will be removed in release 1.12+.
 	SecurePort int
 
+	// MetricsBindAddress is the TCP address that the controller should bind to
+	// for serving prometheus metrics.
+	// It can be set to "0" to disable the metrics serving.
+	// Defaults to ":10351".
+	MetricsBindAddress string
+
+	// HealthProbeBindAddress is the TCP address that the controller should bind to
+	// for serving health probes
+	// It can be set to "0" or "" to disable serving the health probe.
+	// Defaults to ":10351".
+	HealthProbeBindAddress string
+
 	// KubeAPIQPS is the QPS to use while talking with karmada-apiserver.
 	KubeAPIQPS float32
 	// KubeAPIBurst is the burst to allow while talking with karmada-apiserver.
@@ -139,6 +153,12 @@ func (o *Options) AddFlags(fs *pflag.FlagSet) {
 	fs.StringVar(&o.Master, "master", o.Master, "The address of the Kubernetes API server. Overrides any value in KubeConfig. Only required if out-of-cluster.")
 	fs.StringVar(&o.BindAddress, "bind-address", defaultBindAddress, "The IP address on which to listen for the --secure-port port.")
 	fs.IntVar(&o.SecurePort, "secure-port", defaultPort, "The secure port on which to serve HTTPS.")
+	// nolint: errcheck
+	fs.MarkDeprecated("bind-address", "This flag is deprecated and will be removed in release 1.12+. Use --health-probe-bind-address and --metrics-bind-address instead. Note: In release 1.12+, these two addresses cannot be the same.")
+	// nolint: errcheck
+	fs.MarkDeprecated("secure-port", "This flag is deprecated and will be removed in release 1.12+. Use --health-probe-bind-address and --metrics-bind-address instead. Note: In release 1.12+, these two addresses cannot be the same.")
+	fs.StringVar(&o.MetricsBindAddress, "metrics-bind-address", "", "The TCP address that the server should bind to for serving prometheus metrics(e.g. 127.0.0.1:9000, :10351). It can be set to \"0\" to disable the metrics serving. Defaults to 0.0.0.0:10351.")
+	fs.StringVar(&o.HealthProbeBindAddress, "health-probe-bind-address", "", "The TCP address that the server should bind to for serving health probes(e.g. 127.0.0.1:9000, :10351). It can be set to \"0\" to disable serving the health probe. Defaults to 0.0.0.0:10351.")
 	fs.Float32Var(&o.KubeAPIQPS, "kube-api-qps", 40.0, "QPS to use while talking with karmada-apiserver.")
 	fs.IntVar(&o.KubeAPIBurst, "kube-api-burst", 60, "Burst to use while talking with karmada-apiserver.")
 	fs.BoolVar(&o.EnableSchedulerEstimator, "enable-scheduler-estimator", false, "Enable calling cluster scheduler estimator for adjusting replicas.")

cmd/scheduler/app/options/validation.go

@@ -18,27 +18,30 @@ package options
 
 import (
 	"net"
+	"strconv"
 
 	"k8s.io/apimachinery/pkg/util/validation/field"
 )
 
+// Complete ensures that options are valid and marshals them if necessary.
+func (o *Options) Complete() error {
+	if len(o.HealthProbeBindAddress) == 0 {
+		o.HealthProbeBindAddress = net.JoinHostPort(o.BindAddress, strconv.Itoa(o.SecurePort))
+	}
+	if len(o.MetricsBindAddress) == 0 {
+		o.MetricsBindAddress = net.JoinHostPort(o.BindAddress, strconv.Itoa(o.SecurePort))
+	}
+	return nil
+}
+
 // Validate checks Options and return a slice of found errs.
 func (o *Options) Validate() field.ErrorList {
 	errs := field.ErrorList{}
 
 	newPath := field.NewPath("Options")
-	if net.ParseIP(o.BindAddress) == nil {
-		errs = append(errs, field.Invalid(newPath.Child("BindAddress"), o.BindAddress, "not a valid textual representation of an IP address"))
-	}
-
-	if o.SecurePort < 0 || o.SecurePort > 65535 {
-		errs = append(errs, field.Invalid(newPath.Child("SecurePort"), o.SecurePort, "must be a valid port between 0 and 65535 inclusive"))
-	}
-
 	if o.SchedulerEstimatorPort < 0 || o.SchedulerEstimatorPort > 65535 {
 		errs = append(errs, field.Invalid(newPath.Child("SchedulerEstimatorPort"), o.SchedulerEstimatorPort, "must be a valid port between 0 and 65535 inclusive"))
 	}
-
 	if o.SchedulerEstimatorTimeout.Duration < 0 {
 		errs = append(errs, field.Invalid(newPath.Child("SchedulerEstimatorTimeout"), o.SchedulerEstimatorTimeout, "must be greater than or equal to 0"))
 	}

cmd/scheduler/app/options/validation_test.go

@@ -62,8 +62,6 @@ func TestValidateKarmadaSchedulerConfiguration(t *testing.T) {
 			LeaderElection: componentbaseconfig.LeaderElectionConfiguration{
 				LeaderElect: false,
 			},
-			BindAddress:   "127.0.0.1",
-			SecurePort:    9000,
 			KubeAPIQPS:    40,
 			KubeAPIBurst:  30,
 			SchedulerName: "default-scheduler",
@@ -81,18 +79,6 @@ func TestValidateKarmadaSchedulerConfiguration(t *testing.T) {
 		opt          Options
 		expectedErrs field.ErrorList
 	}{
-		"invalid BindAddress": {
-			opt: New(func(option *Options) {
-				option.BindAddress = "127.0.0.1:8080"
-			}),
-			expectedErrs: field.ErrorList{field.Invalid(newPath.Child("BindAddress"), "127.0.0.1:8080", "not a valid textual representation of an IP address")},
-		},
-		"invalid SecurePort": {
-			opt: New(func(option *Options) {
-				option.SecurePort = 90000
-			}),
-			expectedErrs: field.ErrorList{field.Invalid(newPath.Child("SecurePort"), 90000, "must be a valid port between 0 and 65535 inclusive")},
-		},
 		"invalid SchedulerEstimatorPort": {
 			opt: New(func(option *Options) {
 				option.SchedulerEstimatorPort = 90000

cmd/scheduler/app/scheduler.go

@@ -19,10 +19,8 @@ package app
 import (
 	"context"
 	"fmt"
-	"net"
 	"net/http"
 	"os"
-	"strconv"
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus/promhttp"
@@ -100,6 +98,10 @@ The scheduler determines which clusters are valid placements for each resource i
 constraints and available resources. The scheduler then ranks each valid cluster and binds the resource to
 the most suitable cluster.`,
 		RunE: func(_ *cobra.Command, _ []string) error {
+			// complete options
+			if err := opts.Complete(); err != nil {
+				return err
+			}
 			// validate options
 			if errs := opts.Validate(); len(errs) != 0 {
 				return errs.ToAggregate()
@@ -139,7 +141,7 @@ the most suitable cluster.`,
 
 func run(opts *options.Options, stopChan <-chan struct{}, registryOptions ...Option) error {
 	klog.Infof("karmada-scheduler version: %s", version.Get())
-	go serveHealthzAndMetrics(net.JoinHostPort(opts.BindAddress, strconv.Itoa(opts.SecurePort)))
+	serveHealthzAndMetrics(opts.HealthProbeBindAddress, opts.MetricsBindAddress)
 
 	profileflag.ListenAndServe(opts.ProfileOpts)
 
@@ -225,26 +227,64 @@ func run(opts *options.Options, stopChan <-chan struct{}, registryOptions ...Opt
 	return nil
 }
 
-func serveHealthzAndMetrics(address string) {
+func serveHealthzAndMetrics(healthProbeBindAddress, metricsBindAddress string) {
+	if healthProbeBindAddress == metricsBindAddress {
+		if healthProbeBindAddress != "0" {
+			go serveCombined(healthProbeBindAddress)
+		}
+	} else {
+		if healthProbeBindAddress != "0" {
+			go serveHealthz(healthProbeBindAddress)
+		}
+		if metricsBindAddress != "0" {
+			go serveMetrics(metricsBindAddress)
+		}
+	}
+}
+
+func serveCombined(address string) {
 	mux := http.NewServeMux()
-	mux.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) {
+	mux.HandleFunc("/healthz", healthzHandler)
-		w.WriteHeader(http.StatusOK)
+	mux.Handle("/metrics", metricsHandler())
-		_, _ = w.Write([]byte("ok"))
-	})
 
-	mux.Handle("/metrics", promhttp.HandlerFor(ctrlmetrics.Registry, promhttp.HandlerOpts{
+	serveHTTP(address, mux, "healthz and metrics")
+}
+
+func serveHealthz(address string) {
+	mux := http.NewServeMux()
+	mux.HandleFunc("/healthz", healthzHandler)
+	serveHTTP(address, mux, "healthz")
+}
+
+func serveMetrics(address string) {
+	mux := http.NewServeMux()
+	mux.Handle("/metrics", metricsHandler())
+	serveHTTP(address, mux, "metrics")
+}
+
+func healthzHandler(w http.ResponseWriter, _ *http.Request) {
+	w.WriteHeader(http.StatusOK)
+	_, _ = w.Write([]byte("ok"))
+}
+
+func metricsHandler() http.Handler {
+	return promhttp.HandlerFor(ctrlmetrics.Registry, promhttp.HandlerOpts{
 		ErrorHandling: promhttp.HTTPErrorOnError,
-	}))
+	})
+}
 
-	httpServer := http.Server{
+func serveHTTP(address string, handler http.Handler, name string) {
+	httpServer := &http.Server{
 		Addr:              address,
-		Handler:           mux,
+		Handler:           handler,
 		ReadHeaderTimeout: ReadHeaderTimeout,
 		WriteTimeout:      WriteTimeout,
 		ReadTimeout:       ReadTimeout,
 	}
+
+	klog.Infof("Starting %s server on %s", name, address)
 	if err := httpServer.ListenAndServe(); err != nil {
-		klog.Errorf("Failed to serve healthz and metrics: %v", err)
+		klog.Errorf("Failed to serve %s on %s: %v", name, address, err)
 		os.Exit(1)
 	}
 }

operator/pkg/controlplane/manifests.go

@@ -183,8 +183,8 @@ spec:
         command:
         - /bin/karmada-scheduler
         - --kubeconfig=/etc/karmada/kubeconfig
-        - --bind-address=0.0.0.0
+        - --metrics-bind-address=0.0.0.0:10351
-        - --secure-port=10351
+        - --health-probe-bind-address=0.0.0.0:10351
         - --enable-scheduler-estimator=true
         - --leader-elect-resource-namespace={{ .SystemNamespace }}
         - --scheduler-estimator-ca-file=/etc/karmada/pki/ca.crt

pkg/karmadactl/cmdinit/kubernetes/deployments.go

@@ -452,8 +452,8 @@ func (i *CommandInitOption) makeKarmadaSchedulerDeployment() *appsv1.Deployment
 				Command: []string{
 					"/bin/karmada-scheduler",
 					"--kubeconfig=/etc/kubeconfig",
-					"--bind-address=0.0.0.0",
+					"--metrics-bind-address=0.0.0.0:10351",
-					"--secure-port=10351",
+					"--health-probe-bind-address=0.0.0.0:10351",
 					"--enable-scheduler-estimator=true",
 					"--leader-elect=true",
 					"--scheduler-estimator-ca-file=/etc/karmada/pki/ca.crt",