Merge pull request #5290 from seanlaii/controller-manager-options

Standardize the health probe argument of karmada-controller-manager

artifacts/deploy/karmada-controller-manager.yaml

@@ -26,12 +26,11 @@ spec:
           command:
             - /bin/karmada-controller-manager
             - --kubeconfig=/etc/kubeconfig
-            - --bind-address=0.0.0.0
             - --cluster-status-update-frequency=10s
-            - --secure-port=10357
             - --failover-eviction-timeout=30s
             - --controllers=*,hpaScaleTargetMarker,deploymentReplicasSyncer
             - --feature-gates=PropagationPolicyPreemption=true,MultiClusterService=true
+            - --health-probe-bind-address=0.0.0.0:10357
             - --v=4
           livenessProbe:
             httpGet:

cmd/controller-manager/app/controllermanager.go

@@ -19,8 +19,6 @@ package app
 import (
 	"context"
 	"flag"
-	"net"
-	"strconv"
 	"time"
 
 	"github.com/spf13/cobra"
@@ -102,6 +100,10 @@ func NewControllerManagerCommand(ctx context.Context) *cobra.Command {
 The controllers watch Karmada objects and then talk to the underlying clusters' API servers
 to create regular Kubernetes resources.`,
 		RunE: func(_ *cobra.Command, _ []string) error {
+			// complete options
+			if err := opts.Complete(); err != nil {
+				return err
+			}
 			// validate options
 			if errs := opts.Validate(); len(errs) != 0 {
 				return errs.ToAggregate()
@@ -156,7 +158,7 @@ func Run(ctx context.Context, opts *options.Options) error {
 		RenewDeadline:              &opts.LeaderElection.RenewDeadline.Duration,
 		RetryPeriod:                &opts.LeaderElection.RetryPeriod.Duration,
 		LeaderElectionResourceLock: opts.LeaderElection.ResourceLock,
-		HealthProbeBindAddress:     net.JoinHostPort(opts.BindAddress, strconv.Itoa(opts.SecurePort)),
+		HealthProbeBindAddress:     opts.HealthProbeBindAddress,
 		LivenessEndpointName:       "/healthz",
 		Metrics:                    metricsserver.Options{BindAddress: opts.MetricsBindAddress},
 		MapperProvider:             restmapper.MapperProvider,

cmd/controller-manager/app/options/options.go

@@ -18,7 +18,9 @@ package options
 
 import (
 	"fmt"
+	"net"
 	"regexp"
+	"strconv"
 	"strings"
 	"time"
 
@@ -56,9 +58,11 @@ type Options struct {
 	// LeaderElection defines the configuration of leader election client.
 	LeaderElection componentbaseconfig.LeaderElectionConfiguration
 	// BindAddress is the IP address on which to listen for the --secure-port port.
+	// Deprecated: Use HealthProbeBindAddress instead. This will be removed in release 1.12+.
 	BindAddress string
 	// SecurePort is the port that the the server serves at.
 	// Note: We hope support https in the future once controller-runtime provides the functionality.
+	// Deprecated: Use HealthProbeBindAddress instead. This will be removed in release 1.12+.
 	SecurePort int
 	// ClusterStatusUpdateFrequency is the frequency that controller computes and report cluster status.
 	// It must work with ClusterMonitorGracePeriod(--cluster-monitor-grace-period) in karmada-controller-manager.
@@ -112,6 +116,11 @@ type Options struct {
 	// It can be set to "0" to disable the metrics serving.
 	// Defaults to ":8080".
 	MetricsBindAddress string
+	// HealthProbeBindAddress is the TCP address that the controller should bind to
+	// for serving health probes
+	// It can be set to "0" to disable serving the health probe.
+	// Defaults to ":10357".
+	HealthProbeBindAddress string
 	// ConcurrentClusterSyncs is the number of cluster objects that are
 	// allowed to sync concurrently.
 	ConcurrentClusterSyncs int
@@ -172,6 +181,10 @@ func (o *Options) AddFlags(flags *pflag.FlagSet, allControllers, disabledByDefau
 		"The IP address on which to listen for the --secure-port port.")
 	flags.IntVar(&o.SecurePort, "secure-port", defaultPort,
 		"The secure port on which to serve HTTPS.")
+	// nolint: errcheck
+	flags.MarkDeprecated("bind-address", "This flag is deprecated and will be removed in release 1.12+. Use --health-probe-bind-address instead.")
+	// nolint: errcheck
+	flags.MarkDeprecated("secure-port", "This flag is deprecated and will be removed in release 1.12+. Use --health-probe-bind-address instead.")
 	flags.DurationVar(&o.ClusterStatusUpdateFrequency.Duration, "cluster-status-update-frequency", 10*time.Second,
 		"Specifies how often karmada-controller-manager posts cluster status to karmada-apiserver.")
 	flags.BoolVar(&o.LeaderElection.LeaderElect, "leader-elect", true, "Start a leader election client and gain leadership before executing the main loop. Enable this when running replicated components for high availability.")
@@ -219,6 +232,7 @@ func (o *Options) AddFlags(flags *pflag.FlagSet, allControllers, disabledByDefau
 	flags.DurationVar(&o.ClusterCacheSyncTimeout.Duration, "cluster-cache-sync-timeout", util.CacheSyncTimeout, "Timeout period waiting for cluster cache to sync.")
 	flags.DurationVar(&o.ResyncPeriod.Duration, "resync-period", 0, "Base frequency the informers are resynced.")
 	flags.StringVar(&o.MetricsBindAddress, "metrics-bind-address", ":8080", "The TCP address that the controller should bind to for serving prometheus metrics(e.g. 127.0.0.1:8080, :8080). It can be set to \"0\" to disable the metrics serving.")
+	flags.StringVar(&o.HealthProbeBindAddress, "health-probe-bind-address", "", "The TCP address that the controller should bind to for serving health probes(e.g. 127.0.0.1:10357, :10357). It can be set to \"0\" to disable serving the health probe. Defaults to 0.0.0.0:10357.")
 	flags.IntVar(&o.ConcurrentClusterSyncs, "concurrent-cluster-syncs", 5, "The number of Clusters that are allowed to sync concurrently.")
 	flags.IntVar(&o.ConcurrentClusterResourceBindingSyncs, "concurrent-clusterresourcebinding-syncs", 5, "The number of ClusterResourceBindings that are allowed to sync concurrently.")
 	flags.IntVar(&o.ConcurrentResourceBindingSyncs, "concurrent-resourcebinding-syncs", 5, "The number of ResourceBindings that are allowed to sync concurrently.")
@@ -247,3 +261,11 @@ func (o *Options) SkippedNamespacesRegexps() []*regexp.Regexp {
 	}
 	return skippedPropagatingNamespaces
 }
+
+// Complete ensures that options are valid and marshals them if necessary.
+func (o *Options) Complete() error {
+	if len(o.HealthProbeBindAddress) == 0 {
+		o.HealthProbeBindAddress = net.JoinHostPort(o.BindAddress, strconv.Itoa(o.SecurePort))
+	}
+	return nil
+}

cmd/controller-manager/app/options/validation.go

@@ -34,9 +34,6 @@ func (o *Options) Validate() field.ErrorList {
 	if err := skippedResourceConfig.Parse(o.SkippedPropagatingAPIs); err != nil {
 		errs = append(errs, field.Invalid(newPath.Child("SkippedPropagatingAPIs"), o.SkippedPropagatingAPIs, "Invalid API string"))
 	}
-	if o.SecurePort < 0 || o.SecurePort > 65535 {
-		errs = append(errs, field.Invalid(newPath.Child("SecurePort"), o.SecurePort, "must be between 0 and 65535 inclusive"))
-	}
 	if o.ClusterStatusUpdateFrequency.Duration <= 0 {
 		errs = append(errs, field.Invalid(newPath.Child("ClusterStatusUpdateFrequency"), o.ClusterStatusUpdateFrequency, "must be greater than 0"))
 	}

cmd/controller-manager/app/options/validation_test.go

@@ -31,7 +31,6 @@ type ModifyOptions func(option *Options)
 func New(modifyOptions ModifyOptions) Options {
 	option := Options{
 		SkippedPropagatingAPIs:       "cluster.karmada.io;policy.karmada.io;work.karmada.io",
-		SecurePort:                   8090,
 		ClusterStatusUpdateFrequency: metav1.Duration{Duration: 10 * time.Second},
 		ClusterLeaseDuration:         metav1.Duration{Duration: 10 * time.Second},
 		ClusterMonitorPeriod:         metav1.Duration{Duration: 10 * time.Second},
@@ -67,12 +66,6 @@ func TestValidateControllerManagerConfiguration(t *testing.T) {
 			}),
 			expectedErrs: field.ErrorList{field.Invalid(newPath.Child("SkippedPropagatingAPIs"), "a/b/c/d?", "Invalid API string")},
 		},
-		"invalid SecurePort": {
-			opt: New(func(options *Options) {
-				options.SecurePort = -10
-			}),
-			expectedErrs: field.ErrorList{field.Invalid(newPath.Child("SecurePort"), -10, "must be between 0 and 65535 inclusive")},
-		},
 		"invalid ClusterStatusUpdateFrequency": {
 			opt: New(func(options *Options) {
 				options.ClusterStatusUpdateFrequency.Duration = -10 * time.Second

operator/pkg/controlplane/manifests.go

@@ -128,11 +128,10 @@ spec:
         command:
         - /bin/karmada-controller-manager
         - --kubeconfig=/etc/karmada/kubeconfig
-        - --bind-address=0.0.0.0
         - --cluster-status-update-frequency=10s
-        - --secure-port=10357
         - --failover-eviction-timeout=30s
         - --leader-elect-resource-namespace={{ .SystemNamespace }}
+        - --health-probe-bind-address=0.0.0.0:10357
         - --v=4
         livenessProbe:
           httpGet:

pkg/karmadactl/cmdinit/kubernetes/deployments.go

@@ -591,10 +591,9 @@ func (i *CommandInitOption) makeKarmadaControllerManagerDeployment() *appsv1.Dep
 				Command: []string{
 					"/bin/karmada-controller-manager",
 					"--kubeconfig=/etc/kubeconfig",
-					"--bind-address=0.0.0.0",
 					"--metrics-bind-address=:8080",
+					"--health-probe-bind-address=0.0.0.0:10357",
 					"--cluster-status-update-frequency=10s",
-					"--secure-port=10357",
 					fmt.Sprintf("--leader-elect-resource-namespace=%s", i.Namespace),
 					"--v=4",
 				},