Merge pull request #5435 from whitewindmills/descheduler-options

Standardize the health probe and metrics arguments of descheduler

artifacts/deploy/karmada-descheduler.yaml

@@ -26,7 +26,8 @@ spec:
           command:
             - /bin/karmada-descheduler
             - --kubeconfig=/etc/kubeconfig
-            - --bind-address=0.0.0.0
+            - --metrics-bind-address=0.0.0.0:10358
+            - --health-probe-bind-address=0.0.0.0:10358
             - --scheduler-estimator-ca-file=/etc/karmada/pki/ca.crt
             - --scheduler-estimator-cert-file=/etc/karmada/pki/karmada.crt
             - --scheduler-estimator-key-file=/etc/karmada/pki/karmada.key

charts/karmada/templates/karmada-descheduler.yaml

@@ -50,7 +50,8 @@ spec:
           command:
             - /bin/karmada-descheduler
             - --kubeconfig=/etc/kubeconfig
-            - --bind-address=0.0.0.0
+            - --metrics-bind-address=0.0.0.0:10358
+            - --health-probe-bind-address=0.0.0.0:10358
             - --leader-elect-resource-namespace={{ $systemNamespace }}
             - --scheduler-estimator-ca-file=/etc/karmada/pki/server-ca.crt
             - --scheduler-estimator-cert-file=/etc/karmada/pki/karmada.crt

cmd/descheduler/app/descheduler.go

@@ -19,10 +19,8 @@ package app
 import (
 	"context"
 	"fmt"
-	"net"
 	"net/http"
 	"os"
-	"strconv"
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus/promhttp"
@@ -87,6 +85,11 @@ func NewDeschedulerCommand(stopChan <-chan struct{}) *cobra.Command {
 if they are failed to be scheduled for a period of time. It relies on 
 karmada-scheduler-estimator to get replica status.`,
 		RunE: func(_ *cobra.Command, _ []string) error {
+			// complete options
+			if err := opts.Complete(); err != nil {
+				return err
+			}
+
 			// validate options
 			if errs := opts.Validate(); len(errs) != 0 {
 				return errs.ToAggregate()
@@ -127,7 +130,7 @@ karmada-scheduler-estimator to get replica status.`,
 func run(opts *options.Options, stopChan <-chan struct{}) error {
 	klog.Infof("karmada-descheduler version: %s", version.Get())
 	klog.Infof("Please make sure the karmada-scheduler-estimator of all member clusters has been deployed")
-	go serveHealthzAndMetrics(net.JoinHostPort(opts.BindAddress, strconv.Itoa(opts.SecurePort)))
+	serveHealthzAndMetrics(opts.HealthProbeBindAddress, opts.MetricsBindAddress)
 
 	profileflag.ListenAndServe(opts.ProfileOpts)
 
@@ -191,26 +194,64 @@ func run(opts *options.Options, stopChan <-chan struct{}) error {
 	return nil
 }
 
-func serveHealthzAndMetrics(address string) {
+func serveHealthzAndMetrics(healthProbeBindAddress, metricsBindAddress string) {
+	if healthProbeBindAddress == metricsBindAddress {
+		if healthProbeBindAddress != "0" {
+			go serveCombined(healthProbeBindAddress)
+		}
+	} else {
+		if healthProbeBindAddress != "0" {
+			go serveHealthz(healthProbeBindAddress)
+		}
+		if metricsBindAddress != "0" {
+			go serveMetrics(metricsBindAddress)
+		}
+	}
+}
+
+func serveCombined(address string) {
 	mux := http.NewServeMux()
-	mux.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) {
+	mux.HandleFunc("/healthz", healthzHandler)
+	mux.Handle("/metrics", metricsHandler())
+
+	serveHTTP(address, mux, "healthz and metrics")
+}
+
+func serveHealthz(address string) {
+	mux := http.NewServeMux()
+	mux.HandleFunc("/healthz", healthzHandler)
+	serveHTTP(address, mux, "healthz")
+}
+
+func serveMetrics(address string) {
+	mux := http.NewServeMux()
+	mux.Handle("/metrics", metricsHandler())
+	serveHTTP(address, mux, "metrics")
+}
+
+func healthzHandler(w http.ResponseWriter, _ *http.Request) {
 	w.WriteHeader(http.StatusOK)
 	_, _ = w.Write([]byte("ok"))
-	})
+}
 
-	mux.Handle("/metrics", promhttp.HandlerFor(ctrlmetrics.Registry, promhttp.HandlerOpts{
+func metricsHandler() http.Handler {
+	return promhttp.HandlerFor(ctrlmetrics.Registry, promhttp.HandlerOpts{
 		ErrorHandling: promhttp.HTTPErrorOnError,
-	}))
+	})
+}
 
-	httpServer := http.Server{
+func serveHTTP(address string, handler http.Handler, name string) {
+	httpServer := &http.Server{
 		Addr:              address,
-		Handler:           mux,
+		Handler:           handler,
 		ReadHeaderTimeout: ReadHeaderTimeout,
 		WriteTimeout:      WriteTimeout,
 		ReadTimeout:       ReadTimeout,
 	}
+
+	klog.Infof("Starting %s server on %s", name, address)
 	if err := httpServer.ListenAndServe(); err != nil {
-		klog.Errorf("Failed to serve healthz and metrics: %v", err)
+		klog.Errorf("Failed to serve %s on %s: %v", name, address, err)
 		os.Exit(1)
 	}
 }

cmd/descheduler/app/options/options.go

@@ -17,6 +17,8 @@ limitations under the License.
 package options
 
 import (
+	"net"
+	"strconv"
 	"time"
 
 	"github.com/spf13/pflag"
@@ -48,8 +50,10 @@ type Options struct {
 	KubeConfig     string
 	Master         string
 	// BindAddress is the IP address on which to listen for the --secure-port port.
+	// Deprecated: To specify the TCP address for serving health probes, use HealthProbeBindAddress instead. To specify the TCP address for serving prometheus metrics, use MetricsBindAddress instead. This will be removed in release 1.12+.
 	BindAddress string
 	// SecurePort is the port that the server serves at.
+	// Deprecated: To specify the TCP address for serving health probes, use HealthProbeBindAddress instead. To specify the TCP address for serving prometheus metrics, use MetricsBindAddress instead. This will be removed in release 1.12+.
 	SecurePort int
 
 	KubeAPIQPS float32
@@ -75,6 +79,16 @@ type Options struct {
 	// UnschedulableThreshold specifies the period of pod unschedulable condition.
 	UnschedulableThreshold metav1.Duration
 	ProfileOpts            profileflag.Options
+	// MetricsBindAddress is the TCP address that the server should bind to
+	// for serving prometheus metrics.
+	// It can be set to "0" to disable the metrics serving.
+	// Defaults to ":10358".
+	MetricsBindAddress string
+	// HealthProbeBindAddress is the TCP address that the server should bind to
+	// for serving health probes
+	// It can be set to "0" to disable serving the health probe.
+	// Defaults to ":10358".
+	HealthProbeBindAddress string
 }
 
 // NewOptions builds a default descheduler options.
@@ -103,6 +117,10 @@ func (o *Options) AddFlags(fs *pflag.FlagSet) {
 	fs.StringVar(&o.Master, "master", o.Master, "The address of the Kubernetes API server. Overrides any value in KubeConfig. Only required if out-of-cluster.")
 	fs.StringVar(&o.BindAddress, "bind-address", defaultBindAddress, "The IP address on which to listen for the --secure-port port.")
 	fs.IntVar(&o.SecurePort, "secure-port", defaultPort, "The secure port on which to serve HTTPS.")
+	// nolint: errcheck
+	fs.MarkDeprecated("bind-address", "This flag is deprecated and will be removed in release 1.12+. Use --health-probe-bind-address and --metrics-bind-address instead. Note: In release 1.12+, these two addresses cannot be the same.")
+	// nolint: errcheck
+	fs.MarkDeprecated("secure-port", "This flag is deprecated and will be removed in release 1.12+. Use --health-probe-bind-address and --metrics-bind-address instead. Note: In release 1.12+, these two addresses cannot be the same.")
 	fs.Float32Var(&o.KubeAPIQPS, "kube-api-qps", 40.0, "QPS to use while talking with karmada-apiserver.")
 	fs.IntVar(&o.KubeAPIBurst, "kube-api-burst", 60, "Burst to use while talking with karmada-apiserver.")
 	fs.DurationVar(&o.SchedulerEstimatorTimeout.Duration, "scheduler-estimator-timeout", 3*time.Second, "Specifies the timeout period of calling the scheduler estimator service.")
@@ -114,5 +132,18 @@ func (o *Options) AddFlags(fs *pflag.FlagSet) {
 	fs.StringVar(&o.SchedulerEstimatorServicePrefix, "scheduler-estimator-service-prefix", "karmada-scheduler-estimator", "The prefix of scheduler estimator service name")
 	fs.DurationVar(&o.DeschedulingInterval.Duration, "descheduling-interval", defaultDeschedulingInterval, "Time interval between two consecutive descheduler executions. Setting this value instructs the descheduler to run in a continuous loop at the interval specified.")
 	fs.DurationVar(&o.UnschedulableThreshold.Duration, "unschedulable-threshold", defaultUnschedulableThreshold, "The period of pod unschedulable condition. This value is considered as a classification standard of unschedulable replicas.")
+	fs.StringVar(&o.MetricsBindAddress, "metrics-bind-address", "", "The TCP address that the server should bind to for serving prometheus metrics(e.g. 127.0.0.1:10358, :10358). It can be set to \"0\" to disable the metrics serving. Defaults to 0.0.0.0:10358.")
+	fs.StringVar(&o.HealthProbeBindAddress, "health-probe-bind-address", "", "The TCP address that the server should bind to for serving health probes(e.g. 127.0.0.1:10358, :10358). It can be set to \"0\" to disable serving the health probe. Defaults to 0.0.0.0:10358.")
 	o.ProfileOpts.AddFlags(fs)
 }
+
+// Complete ensures that options are valid and marshals them if necessary.
+func (o *Options) Complete() error {
+	if len(o.HealthProbeBindAddress) == 0 {
+		o.HealthProbeBindAddress = net.JoinHostPort(o.BindAddress, strconv.Itoa(o.SecurePort))
+	}
+	if len(o.MetricsBindAddress) == 0 {
+		o.MetricsBindAddress = net.JoinHostPort(o.BindAddress, strconv.Itoa(o.SecurePort))
+	}
+	return nil
+}

cmd/descheduler/app/options/validation.go

@@ -17,8 +17,6 @@ limitations under the License.
 package options
 
 import (
-	"net"
-
 	"k8s.io/apimachinery/pkg/util/validation/field"
 )
 
@@ -27,14 +25,6 @@ func (o *Options) Validate() field.ErrorList {
 	errs := field.ErrorList{}
 
 	newPath := field.NewPath("Options")
-	if net.ParseIP(o.BindAddress) == nil {
-		errs = append(errs, field.Invalid(newPath.Child("BindAddress"), o.BindAddress, "not a valid textual representation of an IP address"))
-	}
-
-	if o.SecurePort < 0 || o.SecurePort > 65535 {
-		errs = append(errs, field.Invalid(newPath.Child("SecurePort"), o.SecurePort, "must be a valid port between 0 and 65535 inclusive"))
-	}
-
 	if o.SchedulerEstimatorPort < 0 || o.SchedulerEstimatorPort > 65535 {
 		errs = append(errs, field.Invalid(newPath.Child("SchedulerEstimatorPort"), o.SchedulerEstimatorPort, "must be a valid port between 0 and 65535 inclusive"))
 	}

cmd/descheduler/app/options/validation_test.go

@@ -34,8 +34,6 @@ func New(modifyOptions ModifyOptions) Options {
 		LeaderElection: componentbaseconfig.LeaderElectionConfiguration{
 			LeaderElect: false,
 		},
-		BindAddress:               "127.0.0.1",
-		SecurePort:                9000,
 		KubeAPIQPS:                40,
 		KubeAPIBurst:              30,
 		SchedulerEstimatorTimeout: metav1.Duration{Duration: 1 * time.Second},
@@ -78,18 +76,6 @@ func TestValidateKarmadaDescheduler(t *testing.T) {
 		opt          Options
 		expectedErrs field.ErrorList
 	}{
-		"invalid BindAddress": {
-			opt: New(func(option *Options) {
-				option.BindAddress = "127.0.0.1:8080"
-			}),
-			expectedErrs: field.ErrorList{field.Invalid(newPath.Child("BindAddress"), "127.0.0.1:8080", "not a valid textual representation of an IP address")},
-		},
-		"invalid SecurePort": {
-			opt: New(func(option *Options) {
-				option.SecurePort = 90000
-			}),
-			expectedErrs: field.ErrorList{field.Invalid(newPath.Child("SecurePort"), 90000, "must be a valid port between 0 and 65535 inclusive")},
-		},
 		"invalid SchedulerEstimatorPort": {
 			opt: New(func(option *Options) {
 				option.SchedulerEstimatorPort = 90000

operator/pkg/controlplane/manifests.go

@@ -248,7 +248,8 @@ spec:
         command:
         - /bin/karmada-descheduler
         - --kubeconfig=/etc/karmada/kubeconfig
-        - --bind-address=0.0.0.0
+        - --metrics-bind-address=0.0.0.0:10358
+        - --health-probe-bind-address=0.0.0.0:10358
         - --leader-elect-resource-namespace={{ .SystemNamespace }}
         - --scheduler-estimator-ca-file=/etc/karmada/pki/ca.crt
         - --scheduler-estimator-cert-file=/etc/karmada/pki/karmada.crt

pkg/karmadactl/addons/descheduler/manifests.go

@@ -45,8 +45,8 @@ spec:
           command:
             - /bin/karmada-descheduler
             - --kubeconfig=/etc/kubeconfig
-            - --bind-address=0.0.0.0
-            - --secure-port=10358
+            - --metrics-bind-address=0.0.0.0:10358
+            - --health-probe-bind-address=0.0.0.0:10358
             - --leader-elect-resource-namespace={{ .Namespace }}
             - --scheduler-estimator-ca-file=/etc/karmada/pki/ca.crt
             - --scheduler-estimator-cert-file=/etc/karmada/pki/karmada.crt