From 3b5fc38a41f244d912309fcadd89bfeb24928a66 Mon Sep 17 00:00:00 2001 From: Poor12 Date: Mon, 4 Jul 2022 20:22:25 +0800 Subject: [PATCH] add pprof for all Karmada components Signed-off-by: Poor12 --- cmd/agent/app/agent.go | 4 + cmd/agent/app/options/options.go | 3 + .../app/options/options.go | 19 ++- .../app/options/validation.go | 13 ++ .../app/controllermanager.go | 4 + cmd/controller-manager/app/options/options.go | 3 + cmd/descheduler/app/descheduler.go | 3 + cmd/descheduler/app/options/options.go | 3 + .../app/options/validation_test.go | 151 ++++++------------ cmd/karmada-search/app/options/options.go | 8 + .../app/options/options.go | 4 + .../app/options/validation_test.go | 61 +++---- .../app/scheduler-estimator.go | 3 + cmd/scheduler/app/options/options.go | 3 + cmd/scheduler/app/options/validation_test.go | 116 +++++--------- cmd/scheduler/app/scheduler.go | 3 + cmd/webhook/app/options/options.go | 6 + cmd/webhook/app/options/validation_test.go | 43 ++--- cmd/webhook/app/webhook.go | 4 + pkg/sharedcli/profileflag/profileflag.go | 48 ++++++ 20 files changed, 271 insertions(+), 231 deletions(-) create mode 100644 cmd/aggregated-apiserver/app/options/validation.go create mode 100644 pkg/sharedcli/profileflag/profileflag.go diff --git a/cmd/agent/app/agent.go b/cmd/agent/app/agent.go index 33ca29529..65748a2de 100644 --- a/cmd/agent/app/agent.go +++ b/cmd/agent/app/agent.go @@ -34,6 +34,7 @@ import ( "github.com/karmada-io/karmada/pkg/resourceinterpreter" "github.com/karmada-io/karmada/pkg/sharedcli" "github.com/karmada-io/karmada/pkg/sharedcli/klogflag" + "github.com/karmada-io/karmada/pkg/sharedcli/profileflag" "github.com/karmada-io/karmada/pkg/util" "github.com/karmada-io/karmada/pkg/util/gclient" "github.com/karmada-io/karmada/pkg/util/helper" @@ -104,6 +105,9 @@ func init() { func run(ctx context.Context, karmadaConfig karmadactl.KarmadaConfig, opts *options.Options) error { klog.Infof("karmada-agent version: %s", version.Get()) + + profileflag.ListenAndServe(opts.ProfileOpts) + controlPlaneRestConfig, err := karmadaConfig.GetRestConfig(opts.KarmadaContext, opts.KarmadaKubeConfig) if err != nil { return fmt.Errorf("error building kubeconfig of karmada control plane: %w", err) diff --git a/cmd/agent/app/options/options.go b/cmd/agent/app/options/options.go index d7320c0f9..86e295077 100644 --- a/cmd/agent/app/options/options.go +++ b/cmd/agent/app/options/options.go @@ -10,6 +10,7 @@ import ( "k8s.io/client-go/tools/leaderelection/resourcelock" componentbaseconfig "k8s.io/component-base/config" + "github.com/karmada-io/karmada/pkg/sharedcli/profileflag" "github.com/karmada-io/karmada/pkg/sharedcli/ratelimiterflag" "github.com/karmada-io/karmada/pkg/util" ) @@ -88,6 +89,7 @@ type Options struct { MetricsBindAddress string RateLimiterOpts ratelimiterflag.Options + ProfileOpts profileflag.Options } // NewOptions builds an default scheduler options. @@ -153,4 +155,5 @@ func (o *Options) AddFlags(fs *pflag.FlagSet, allControllers []string) { fs.IntVar(&o.ConcurrentWorkSyncs, "concurrent-work-syncs", 5, "The number of Works that are allowed to sync concurrently.") fs.StringVar(&o.MetricsBindAddress, "metrics-bind-address", ":8080", "The TCP address that the controller should bind to for serving prometheus metrics(e.g. 127.0.0.1:8088, :8088)") o.RateLimiterOpts.AddFlags(fs) + o.ProfileOpts.AddFlags(fs) } diff --git a/cmd/aggregated-apiserver/app/options/options.go b/cmd/aggregated-apiserver/app/options/options.go index e8c6c4d0e..a0fdebd22 100644 --- a/cmd/aggregated-apiserver/app/options/options.go +++ b/cmd/aggregated-apiserver/app/options/options.go @@ -10,7 +10,6 @@ import ( "github.com/spf13/pflag" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" - utilerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apiserver/pkg/admission" "k8s.io/apiserver/pkg/endpoints/openapi" @@ -21,6 +20,7 @@ import ( genericoptions "k8s.io/apiserver/pkg/server/options" utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/kubernetes" + "k8s.io/klog/v2" netutils "k8s.io/utils/net" "github.com/karmada-io/karmada/pkg/aggregatedapiserver" @@ -28,7 +28,9 @@ import ( clientset "github.com/karmada-io/karmada/pkg/generated/clientset/versioned" informers "github.com/karmada-io/karmada/pkg/generated/informers/externalversions" generatedopenapi "github.com/karmada-io/karmada/pkg/generated/openapi" + "github.com/karmada-io/karmada/pkg/sharedcli/profileflag" "github.com/karmada-io/karmada/pkg/util/lifted" + "github.com/karmada-io/karmada/pkg/version" ) const defaultEtcdPathPrefix = "/registry" @@ -42,6 +44,8 @@ type Options struct { KubeAPIQPS float32 // KubeAPIBurst is the burst to allow while talking with karmada-apiserver. KubeAPIBurst int + + ProfileOpts profileflag.Options } // NewOptions returns a new Options. @@ -63,6 +67,7 @@ func (o *Options) AddFlags(flags *pflag.FlagSet) { flags.Float32Var(&o.KubeAPIQPS, "kube-api-qps", 40.0, "QPS to use while talking with karmada-apiserver. Doesn't cover events and node heartbeat apis which rate limiting is controlled by a different set of flags.") flags.IntVar(&o.KubeAPIBurst, "kube-api-burst", 60, "Burst to use while talking with karmada-apiserver. Doesn't cover events and node heartbeat apis which rate limiting is controlled by a different set of flags.") utilfeature.DefaultMutableFeatureGate.AddFlag(flags) + o.ProfileOpts.AddFlags(flags) } // Complete fills in fields required to have valid data. @@ -70,15 +75,12 @@ func (o *Options) Complete() error { return nil } -// Validate validates Options. -func (o *Options) Validate() error { - var errs []error - errs = append(errs, o.RecommendedOptions.Validate()...) - return utilerrors.NewAggregate(errs) -} - // Run runs the aggregated-apiserver with options. This should never exit. func (o *Options) Run(ctx context.Context) error { + klog.Infof("karmada-aggregated-apiserver version: %s", version.Get()) + + profileflag.ListenAndServe(o.ProfileOpts) + config, err := o.Config() if err != nil { return err @@ -120,6 +122,7 @@ func (o *Options) Config() (*aggregatedapiserver.Config, error) { o.SharedInformerFactory = informerFactory return []admission.PluginInitializer{}, nil } + o.RecommendedOptions.Features = &genericoptions.FeatureOptions{EnableProfiling: false} serverConfig := genericapiserver.NewRecommendedConfig(aggregatedapiserver.Codecs) serverConfig.LongRunningFunc = customLongRunningRequestCheck(sets.NewString("watch", "proxy"), diff --git a/cmd/aggregated-apiserver/app/options/validation.go b/cmd/aggregated-apiserver/app/options/validation.go new file mode 100644 index 000000000..610c832c5 --- /dev/null +++ b/cmd/aggregated-apiserver/app/options/validation.go @@ -0,0 +1,13 @@ +package options + +import ( + utilerrors "k8s.io/apimachinery/pkg/util/errors" +) + +// Validate validates Options. +func (o *Options) Validate() error { + var errs []error + errs = append(errs, o.RecommendedOptions.Validate()...) + + return utilerrors.NewAggregate(errs) +} diff --git a/cmd/controller-manager/app/controllermanager.go b/cmd/controller-manager/app/controllermanager.go index 16b53fa04..d6b50072a 100644 --- a/cmd/controller-manager/app/controllermanager.go +++ b/cmd/controller-manager/app/controllermanager.go @@ -44,6 +44,7 @@ import ( "github.com/karmada-io/karmada/pkg/resourceinterpreter" "github.com/karmada-io/karmada/pkg/sharedcli" "github.com/karmada-io/karmada/pkg/sharedcli/klogflag" + "github.com/karmada-io/karmada/pkg/sharedcli/profileflag" "github.com/karmada-io/karmada/pkg/util" "github.com/karmada-io/karmada/pkg/util/gclient" "github.com/karmada-io/karmada/pkg/util/helper" @@ -97,6 +98,9 @@ func NewControllerManagerCommand(ctx context.Context) *cobra.Command { // Run runs the controller-manager with options. This should never exit. func Run(ctx context.Context, opts *options.Options) error { klog.Infof("karmada-controller-manager version: %s", version.Get()) + + profileflag.ListenAndServe(opts.ProfileOpts) + config, err := controllerruntime.GetConfig() if err != nil { panic(err) diff --git a/cmd/controller-manager/app/options/options.go b/cmd/controller-manager/app/options/options.go index 2b0e6ef75..082eee36b 100644 --- a/cmd/controller-manager/app/options/options.go +++ b/cmd/controller-manager/app/options/options.go @@ -11,6 +11,7 @@ import ( componentbaseconfig "k8s.io/component-base/config" "github.com/karmada-io/karmada/pkg/features" + "github.com/karmada-io/karmada/pkg/sharedcli/profileflag" "github.com/karmada-io/karmada/pkg/sharedcli/ratelimiterflag" "github.com/karmada-io/karmada/pkg/util" ) @@ -112,6 +113,7 @@ type Options struct { ConcurrentResourceTemplateSyncs int RateLimiterOpts ratelimiterflag.Options + ProfileOpts profileflag.Options } // NewOptions builds an empty options. @@ -190,5 +192,6 @@ func (o *Options) AddFlags(flags *pflag.FlagSet, allControllers, disabledByDefau flags.IntVar(&o.ConcurrentResourceTemplateSyncs, "concurrent-resource-template-syncs", 5, "The number of resource templates that are allowed to sync concurrently.") o.RateLimiterOpts.AddFlags(flags) + o.ProfileOpts.AddFlags(flags) features.FeatureGate.AddFlag(flags) } diff --git a/cmd/descheduler/app/descheduler.go b/cmd/descheduler/app/descheduler.go index 87163e980..8a5e29a26 100644 --- a/cmd/descheduler/app/descheduler.go +++ b/cmd/descheduler/app/descheduler.go @@ -25,6 +25,7 @@ import ( karmadaclientset "github.com/karmada-io/karmada/pkg/generated/clientset/versioned" "github.com/karmada-io/karmada/pkg/sharedcli" "github.com/karmada-io/karmada/pkg/sharedcli/klogflag" + "github.com/karmada-io/karmada/pkg/sharedcli/profileflag" "github.com/karmada-io/karmada/pkg/version" "github.com/karmada-io/karmada/pkg/version/sharedcommand" ) @@ -81,6 +82,8 @@ func run(opts *options.Options, stopChan <-chan struct{}) error { klog.Infof("Please make sure the karmada-scheduler-estimator of all member clusters has been deployed") go serveHealthzAndMetrics(net.JoinHostPort(opts.BindAddress, strconv.Itoa(opts.SecurePort))) + profileflag.ListenAndServe(opts.ProfileOpts) + restConfig, err := clientcmd.BuildConfigFromFlags(opts.Master, opts.KubeConfig) if err != nil { return fmt.Errorf("error building kubeconfig: %s", err.Error()) diff --git a/cmd/descheduler/app/options/options.go b/cmd/descheduler/app/options/options.go index 87eeda053..dcb01d6de 100644 --- a/cmd/descheduler/app/options/options.go +++ b/cmd/descheduler/app/options/options.go @@ -8,6 +8,7 @@ import ( "k8s.io/client-go/tools/leaderelection/resourcelock" componentbaseconfig "k8s.io/component-base/config" + "github.com/karmada-io/karmada/pkg/sharedcli/profileflag" "github.com/karmada-io/karmada/pkg/util" ) @@ -47,6 +48,7 @@ type Options struct { DeschedulingInterval metav1.Duration // UnschedulableThreshold specifies the period of pod unschedulable condition. UnschedulableThreshold metav1.Duration + ProfileOpts profileflag.Options } // NewOptions builds a default descheduler options. @@ -81,4 +83,5 @@ func (o *Options) AddFlags(fs *pflag.FlagSet) { fs.IntVar(&o.SchedulerEstimatorPort, "scheduler-estimator-port", defaultEstimatorPort, "The secure port on which to connect the accurate scheduler estimator.") fs.DurationVar(&o.DeschedulingInterval.Duration, "descheduling-interval", defaultDeschedulingInterval, "Time interval between two consecutive descheduler executions. Setting this value instructs the descheduler to run in a continuous loop at the interval specified.") fs.DurationVar(&o.UnschedulableThreshold.Duration, "unschedulable-threshold", defaultUnschedulableThreshold, "The period of pod unschedulable condition. This value is considered as a classification standard of unschedulable replicas.") + o.ProfileOpts.AddFlags(fs) } diff --git a/cmd/descheduler/app/options/validation_test.go b/cmd/descheduler/app/options/validation_test.go index 3407874ee..d29d575bf 100644 --- a/cmd/descheduler/app/options/validation_test.go +++ b/cmd/descheduler/app/options/validation_test.go @@ -9,34 +9,39 @@ import ( componentbaseconfig "k8s.io/component-base/config" ) +// a callback function to modify options +type ModifyOptions func(option *Options) + +// New an Options with default parameters +func New(modifyOptions ModifyOptions) Options { + option := Options{ + LeaderElection: componentbaseconfig.LeaderElectionConfiguration{ + LeaderElect: false, + }, + BindAddress: "127.0.0.1", + SecurePort: 9000, + KubeAPIQPS: 40, + KubeAPIBurst: 30, + SchedulerEstimatorTimeout: metav1.Duration{Duration: 1 * time.Second}, + SchedulerEstimatorPort: 9001, + DeschedulingInterval: metav1.Duration{Duration: 1 * time.Second}, + UnschedulableThreshold: metav1.Duration{Duration: 1 * time.Second}, + } + + if modifyOptions != nil { + modifyOptions(&option) + } + return option +} + func TestValidateKarmadaDescheduler(t *testing.T) { successCases := []Options{ - { - LeaderElection: componentbaseconfig.LeaderElectionConfiguration{ - LeaderElect: false, - }, - BindAddress: "127.0.0.1", - SecurePort: 9000, - KubeAPIQPS: 40, - KubeAPIBurst: 30, - SchedulerEstimatorTimeout: metav1.Duration{Duration: 1 * time.Second}, - SchedulerEstimatorPort: 9001, - DeschedulingInterval: metav1.Duration{Duration: 1 * time.Second}, - UnschedulableThreshold: metav1.Duration{Duration: 1 * time.Second}, - }, - { - LeaderElection: componentbaseconfig.LeaderElectionConfiguration{ + New(nil), + New(func(option *Options) { + option.LeaderElection = componentbaseconfig.LeaderElectionConfiguration{ LeaderElect: true, - }, - BindAddress: "127.0.0.1", - SecurePort: 9000, - KubeAPIQPS: 40, - KubeAPIBurst: 30, - SchedulerEstimatorTimeout: metav1.Duration{Duration: 1 * time.Second}, - SchedulerEstimatorPort: 9001, - DeschedulingInterval: metav1.Duration{Duration: 1 * time.Second}, - UnschedulableThreshold: metav1.Duration{Duration: 1 * time.Second}, - }, { + } + }), { LeaderElection: componentbaseconfig.LeaderElectionConfiguration{ LeaderElect: false, }, @@ -58,99 +63,39 @@ func TestValidateKarmadaDescheduler(t *testing.T) { expectedErrs field.ErrorList }{ "invalid BindAddress": { - opt: Options{ - LeaderElection: componentbaseconfig.LeaderElectionConfiguration{ - LeaderElect: false, - }, - BindAddress: "127.0.0.1:8080", - SecurePort: 9000, - KubeAPIQPS: 40, - KubeAPIBurst: 30, - SchedulerEstimatorTimeout: metav1.Duration{Duration: 1 * time.Second}, - SchedulerEstimatorPort: 9001, - DeschedulingInterval: metav1.Duration{Duration: 1 * time.Second}, - UnschedulableThreshold: metav1.Duration{Duration: 1 * time.Second}, - }, + opt: New(func(option *Options) { + option.BindAddress = "127.0.0.1:8080" + }), expectedErrs: field.ErrorList{field.Invalid(newPath.Child("BindAddress"), "127.0.0.1:8080", "not a valid textual representation of an IP address")}, }, "invalid SecurePort": { - opt: Options{ - LeaderElection: componentbaseconfig.LeaderElectionConfiguration{ - LeaderElect: false, - }, - BindAddress: "127.0.0.1", - SecurePort: 90000, - KubeAPIQPS: 40, - KubeAPIBurst: 30, - SchedulerEstimatorTimeout: metav1.Duration{Duration: 1 * time.Second}, - SchedulerEstimatorPort: 9001, - DeschedulingInterval: metav1.Duration{Duration: 1 * time.Second}, - UnschedulableThreshold: metav1.Duration{Duration: 1 * time.Second}, - }, + opt: New(func(option *Options) { + option.SecurePort = 90000 + }), expectedErrs: field.ErrorList{field.Invalid(newPath.Child("SecurePort"), 90000, "must be a valid port between 0 and 65535 inclusive")}, }, "invalid SchedulerEstimatorPort": { - opt: Options{ - LeaderElection: componentbaseconfig.LeaderElectionConfiguration{ - LeaderElect: false, - }, - BindAddress: "127.0.0.1", - SecurePort: 9000, - KubeAPIQPS: 40, - KubeAPIBurst: 30, - SchedulerEstimatorTimeout: metav1.Duration{Duration: 1 * time.Second}, - SchedulerEstimatorPort: 90000, - DeschedulingInterval: metav1.Duration{Duration: 1 * time.Second}, - UnschedulableThreshold: metav1.Duration{Duration: 1 * time.Second}, - }, + opt: New(func(option *Options) { + option.SchedulerEstimatorPort = 90000 + }), expectedErrs: field.ErrorList{field.Invalid(newPath.Child("SchedulerEstimatorPort"), 90000, "must be a valid port between 0 and 65535 inclusive")}, }, "invalid SchedulerEstimatorTimeout": { - opt: Options{ - LeaderElection: componentbaseconfig.LeaderElectionConfiguration{ - LeaderElect: false, - }, - BindAddress: "127.0.0.1", - SecurePort: 9000, - KubeAPIQPS: 40, - KubeAPIBurst: 30, - SchedulerEstimatorTimeout: metav1.Duration{Duration: -1 * time.Second}, - SchedulerEstimatorPort: 9000, - DeschedulingInterval: metav1.Duration{Duration: 1 * time.Second}, - UnschedulableThreshold: metav1.Duration{Duration: 1 * time.Second}, - }, + opt: New(func(option *Options) { + option.SchedulerEstimatorTimeout = metav1.Duration{Duration: -1 * time.Second} + }), expectedErrs: field.ErrorList{field.Invalid(newPath.Child("SchedulerEstimatorTimeout"), metav1.Duration{Duration: -1 * time.Second}, "must be greater than or equal to 0")}, }, "invalid DeschedulingInterval": { - opt: Options{ - LeaderElection: componentbaseconfig.LeaderElectionConfiguration{ - LeaderElect: false, - }, - BindAddress: "127.0.0.1", - SecurePort: 9000, - KubeAPIQPS: 40, - KubeAPIBurst: 30, - SchedulerEstimatorTimeout: metav1.Duration{Duration: 1 * time.Second}, - SchedulerEstimatorPort: 9000, - DeschedulingInterval: metav1.Duration{Duration: -1 * time.Second}, - UnschedulableThreshold: metav1.Duration{Duration: 1 * time.Second}, - }, + opt: New(func(option *Options) { + option.DeschedulingInterval = metav1.Duration{Duration: -1 * time.Second} + }), expectedErrs: field.ErrorList{field.Invalid(newPath.Child("DeschedulingInterval"), metav1.Duration{Duration: -1 * time.Second}, "must be greater than or equal to 0")}, }, "invalid UnschedulableThreshold": { - opt: Options{ - LeaderElection: componentbaseconfig.LeaderElectionConfiguration{ - LeaderElect: false, - }, - BindAddress: "127.0.0.1", - SecurePort: 9000, - KubeAPIQPS: 40, - KubeAPIBurst: 30, - SchedulerEstimatorTimeout: metav1.Duration{Duration: 1 * time.Second}, - SchedulerEstimatorPort: 9000, - DeschedulingInterval: metav1.Duration{Duration: 1 * time.Second}, - UnschedulableThreshold: metav1.Duration{Duration: -1 * time.Second}, - }, + opt: New(func(option *Options) { + option.UnschedulableThreshold = metav1.Duration{Duration: -1 * time.Second} + }), expectedErrs: field.ErrorList{field.Invalid(newPath.Child("UnschedulableThreshold"), metav1.Duration{Duration: -1 * time.Second}, "must be greater than or equal to 0")}, }, } diff --git a/cmd/karmada-search/app/options/options.go b/cmd/karmada-search/app/options/options.go index 8ca18d1de..1f4465b96 100644 --- a/cmd/karmada-search/app/options/options.go +++ b/cmd/karmada-search/app/options/options.go @@ -20,6 +20,7 @@ import ( searchv1alpha1 "github.com/karmada-io/karmada/pkg/apis/search/v1alpha1" generatedopenapi "github.com/karmada-io/karmada/pkg/generated/openapi" "github.com/karmada-io/karmada/pkg/search" + "github.com/karmada-io/karmada/pkg/sharedcli/profileflag" "github.com/karmada-io/karmada/pkg/version" ) @@ -33,6 +34,8 @@ type Options struct { KubeAPIQPS float32 // KubeAPIBurst is the burst to allow while talking with karmada-search. KubeAPIBurst int + + ProfileOpts profileflag.Options } // NewOptions returns a new Options. @@ -57,6 +60,7 @@ func (o *Options) AddFlags(flags *pflag.FlagSet) { flags.IntVar(&o.KubeAPIBurst, "kube-api-burst", 60, "Burst to use while talking with karmada-apiserver. Doesn't cover events and node heartbeat apis which rate limiting is controlled by a different set of flags.") utilfeature.DefaultMutableFeatureGate.AddFlag(flags) + o.ProfileOpts.AddFlags(flags) } // Complete fills in fields required to have valid data. @@ -68,6 +72,8 @@ func (o *Options) Complete() error { func (o *Options) Run(ctx context.Context) error { klog.Infof("karmada-search version: %s", version.Get()) + profileflag.ListenAndServe(o.ProfileOpts) + config, err := o.Config() if err != nil { return err @@ -99,6 +105,8 @@ func (o *Options) Config() (*search.Config, error) { return nil, fmt.Errorf("error creating self-signed certificates: %v", err) } + o.RecommendedOptions.Features = &genericoptions.FeatureOptions{EnableProfiling: false} + serverConfig := genericapiserver.NewRecommendedConfig(searchscheme.Codecs) serverConfig.OpenAPIConfig = genericapiserver.DefaultOpenAPIConfig(generatedopenapi.GetOpenAPIDefinitions, openapi.NewDefinitionNamer(searchscheme.Scheme)) serverConfig.OpenAPIConfig.Info.Title = "karmada-search" diff --git a/cmd/scheduler-estimator/app/options/options.go b/cmd/scheduler-estimator/app/options/options.go index 4cfa84cda..1ae61ba0a 100644 --- a/cmd/scheduler-estimator/app/options/options.go +++ b/cmd/scheduler-estimator/app/options/options.go @@ -2,6 +2,8 @@ package options import ( "github.com/spf13/pflag" + + "github.com/karmada-io/karmada/pkg/sharedcli/profileflag" ) const ( @@ -27,6 +29,7 @@ type Options struct { ClusterAPIBurst int // Parallelism defines the amount of parallelism in algorithms for estimating. Must be greater than 0. Defaults to 16. Parallelism int + ProfileOpts profileflag.Options } // NewOptions builds an empty options. @@ -48,4 +51,5 @@ func (o *Options) AddFlags(fs *pflag.FlagSet) { fs.Float32Var(&o.ClusterAPIQPS, "kube-api-qps", 20.0, "QPS to use while talking with apiserver. Doesn't cover events and node heartbeat apis which rate limiting is controlled by a different set of flags.") fs.IntVar(&o.ClusterAPIBurst, "kube-api-burst", 30, "Burst to use while talking with apiserver. Doesn't cover events and node heartbeat apis which rate limiting is controlled by a different set of flags.") fs.IntVar(&o.Parallelism, "parallelism", o.Parallelism, "Parallelism defines the amount of parallelism in algorithms for estimating. Must be greater than 0. Defaults to 16.") + o.ProfileOpts.AddFlags(fs) } diff --git a/cmd/scheduler-estimator/app/options/validation_test.go b/cmd/scheduler-estimator/app/options/validation_test.go index 0940eb86e..8784b8682 100644 --- a/cmd/scheduler-estimator/app/options/validation_test.go +++ b/cmd/scheduler-estimator/app/options/validation_test.go @@ -6,14 +6,27 @@ import ( "k8s.io/apimachinery/pkg/util/validation/field" ) +// a callback function to modify options +type ModifyOptions func(option *Options) + +// New an Options with default parameters +func New(modifyOptions ModifyOptions) Options { + option := Options{ + ClusterName: "testCluster", + BindAddress: "0.0.0.0", + SecurePort: 10100, + ServerPort: 8088, + } + + if modifyOptions != nil { + modifyOptions(&option) + } + return option +} + func TestValidateKarmadaSchedulerEstimator(t *testing.T) { successCases := []Options{ - { - ClusterName: "testCluster", - BindAddress: "0.0.0.0", - SecurePort: 10100, - ServerPort: 8088, - }, + New(nil), } for _, successCase := range successCases { if errs := successCase.Validate(); len(errs) != 0 { @@ -27,39 +40,27 @@ func TestValidateKarmadaSchedulerEstimator(t *testing.T) { expectedErrs field.ErrorList }{ "invalid ClusterName": { - opt: Options{ - ClusterName: "", - BindAddress: "127.0.0.1", - SecurePort: 10100, - ServerPort: 8088, - }, + opt: New(func(option *Options) { + option.ClusterName = "" + }), expectedErrs: field.ErrorList{field.Invalid(newPath.Child("ClusterName"), "", "clusterName cannot be empty")}, }, "invalid BindAddress": { - opt: Options{ - ClusterName: "testCluster", - BindAddress: "127.0.0.1:8082", - SecurePort: 10100, - ServerPort: 8088, - }, + opt: New(func(option *Options) { + option.BindAddress = "127.0.0.1:8082" + }), expectedErrs: field.ErrorList{field.Invalid(newPath.Child("BindAddress"), "127.0.0.1:8082", "not a valid textual representation of an IP address")}, }, "invalid SecurePort": { - opt: Options{ - ClusterName: "testCluster", - BindAddress: "127.0.0.1", - SecurePort: 908188, - ServerPort: 8088, - }, + opt: New(func(option *Options) { + option.SecurePort = 908188 + }), expectedErrs: field.ErrorList{field.Invalid(newPath.Child("SecurePort"), 908188, "must be a valid port between 0 and 65535 inclusive")}, }, "invalid ServerPort": { - opt: Options{ - ClusterName: "testCluster", - BindAddress: "127.0.0.1", - SecurePort: 9089, - ServerPort: 80888, - }, + opt: New(func(option *Options) { + option.ServerPort = 80888 + }), expectedErrs: field.ErrorList{field.Invalid(newPath.Child("ServerPort"), 80888, "must be a valid port between 0 and 65535 inclusive")}, }, } diff --git a/cmd/scheduler-estimator/app/scheduler-estimator.go b/cmd/scheduler-estimator/app/scheduler-estimator.go index 1aaa3fc91..76b2dcb87 100644 --- a/cmd/scheduler-estimator/app/scheduler-estimator.go +++ b/cmd/scheduler-estimator/app/scheduler-estimator.go @@ -21,6 +21,7 @@ import ( "github.com/karmada-io/karmada/pkg/estimator/server" "github.com/karmada-io/karmada/pkg/sharedcli" "github.com/karmada-io/karmada/pkg/sharedcli/klogflag" + "github.com/karmada-io/karmada/pkg/sharedcli/profileflag" "github.com/karmada-io/karmada/pkg/version" "github.com/karmada-io/karmada/pkg/version/sharedcommand" ) @@ -66,6 +67,8 @@ func run(ctx context.Context, opts *options.Options) error { klog.Infof("karmada-scheduler-estimator version: %s", version.Get()) go serveHealthzAndMetrics(net.JoinHostPort(opts.BindAddress, strconv.Itoa(opts.SecurePort))) + profileflag.ListenAndServe(opts.ProfileOpts) + restConfig, err := clientcmd.BuildConfigFromFlags(opts.Master, opts.KubeConfig) if err != nil { return fmt.Errorf("error building kubeconfig: %s", err.Error()) diff --git a/cmd/scheduler/app/options/options.go b/cmd/scheduler/app/options/options.go index 070beecc0..c094c8e2e 100644 --- a/cmd/scheduler/app/options/options.go +++ b/cmd/scheduler/app/options/options.go @@ -9,6 +9,7 @@ import ( componentbaseconfig "k8s.io/component-base/config" "github.com/karmada-io/karmada/pkg/features" + "github.com/karmada-io/karmada/pkg/sharedcli/profileflag" "github.com/karmada-io/karmada/pkg/util" ) @@ -50,6 +51,7 @@ type Options struct { // EnableEmptyWorkloadPropagation represents whether workload with 0 replicas could be propagated to member clusters. EnableEmptyWorkloadPropagation bool + ProfileOpts profileflag.Options } // NewOptions builds an default scheduler options. @@ -87,4 +89,5 @@ func (o *Options) AddFlags(fs *pflag.FlagSet) { fs.IntVar(&o.SchedulerEstimatorPort, "scheduler-estimator-port", defaultEstimatorPort, "The secure port on which to connect the accurate scheduler estimator.") fs.BoolVar(&o.EnableEmptyWorkloadPropagation, "enable-empty-workload-propagation", false, "Enable workload with replicas 0 to be propagated to member clusters.") features.FeatureGate.AddFlag(fs) + o.ProfileOpts.AddFlags(fs) } diff --git a/cmd/scheduler/app/options/validation_test.go b/cmd/scheduler/app/options/validation_test.go index c1aa8ec6e..ba61598cb 100644 --- a/cmd/scheduler/app/options/validation_test.go +++ b/cmd/scheduler/app/options/validation_test.go @@ -9,32 +9,39 @@ import ( componentbaseconfig "k8s.io/component-base/config" ) +// a callback function to modify options +type ModifyOptions func(option *Options) + +// New an Options with default parameters +func New(modifyOptions ModifyOptions) Options { + option := Options{ + LeaderElection: componentbaseconfig.LeaderElectionConfiguration{ + LeaderElect: false, + }, + BindAddress: "127.0.0.1", + SecurePort: 9000, + KubeAPIQPS: 40, + KubeAPIBurst: 30, + EnableSchedulerEstimator: false, + SchedulerEstimatorTimeout: metav1.Duration{Duration: 1 * time.Second}, + SchedulerEstimatorPort: 9001, + } + + if modifyOptions != nil { + modifyOptions(&option) + } + return option +} + func TestValidateKarmadaSchedulerConfiguration(t *testing.T) { successCases := []Options{ - { - LeaderElection: componentbaseconfig.LeaderElectionConfiguration{ - LeaderElect: false, - }, - BindAddress: "127.0.0.1", - SecurePort: 9000, - KubeAPIQPS: 40, - KubeAPIBurst: 30, - EnableSchedulerEstimator: false, - SchedulerEstimatorTimeout: metav1.Duration{Duration: 1 * time.Second}, - SchedulerEstimatorPort: 9001, - }, - { - LeaderElection: componentbaseconfig.LeaderElectionConfiguration{ + New(nil), + New(func(option *Options) { + option.LeaderElection = componentbaseconfig.LeaderElectionConfiguration{ LeaderElect: true, - }, - BindAddress: "127.0.0.1", - SecurePort: 9000, - KubeAPIQPS: 40, - KubeAPIBurst: 30, - EnableSchedulerEstimator: false, - SchedulerEstimatorTimeout: metav1.Duration{Duration: 1 * time.Second}, - SchedulerEstimatorPort: 9001, - }, { + } + }), + { LeaderElection: componentbaseconfig.LeaderElectionConfiguration{ LeaderElect: false, }, @@ -42,7 +49,8 @@ func TestValidateKarmadaSchedulerConfiguration(t *testing.T) { SecurePort: 9000, KubeAPIQPS: 40, KubeAPIBurst: 30, - }} + }, + } for _, successCase := range successCases { if errs := successCase.Validate(); len(errs) != 0 { @@ -56,63 +64,27 @@ func TestValidateKarmadaSchedulerConfiguration(t *testing.T) { expectedErrs field.ErrorList }{ "invalid BindAddress": { - opt: Options{ - LeaderElection: componentbaseconfig.LeaderElectionConfiguration{ - LeaderElect: false, - }, - BindAddress: "127.0.0.1:8080", - SecurePort: 9000, - KubeAPIQPS: 40, - KubeAPIBurst: 30, - EnableSchedulerEstimator: false, - SchedulerEstimatorTimeout: metav1.Duration{Duration: 1 * time.Second}, - SchedulerEstimatorPort: 9001, - }, + opt: New(func(option *Options) { + option.BindAddress = "127.0.0.1:8080" + }), expectedErrs: field.ErrorList{field.Invalid(newPath.Child("BindAddress"), "127.0.0.1:8080", "not a valid textual representation of an IP address")}, }, "invalid SecurePort": { - opt: Options{ - LeaderElection: componentbaseconfig.LeaderElectionConfiguration{ - LeaderElect: false, - }, - BindAddress: "127.0.0.1", - SecurePort: 90000, - KubeAPIQPS: 40, - KubeAPIBurst: 30, - EnableSchedulerEstimator: false, - SchedulerEstimatorTimeout: metav1.Duration{Duration: 1 * time.Second}, - SchedulerEstimatorPort: 9001, - }, + opt: New(func(option *Options) { + option.SecurePort = 90000 + }), expectedErrs: field.ErrorList{field.Invalid(newPath.Child("SecurePort"), 90000, "must be a valid port between 0 and 65535 inclusive")}, }, "invalid SchedulerEstimatorPort": { - opt: Options{ - LeaderElection: componentbaseconfig.LeaderElectionConfiguration{ - LeaderElect: false, - }, - BindAddress: "127.0.0.1", - SecurePort: 9000, - KubeAPIQPS: 40, - KubeAPIBurst: 30, - EnableSchedulerEstimator: false, - SchedulerEstimatorTimeout: metav1.Duration{Duration: 1 * time.Second}, - SchedulerEstimatorPort: 90000, - }, + opt: New(func(option *Options) { + option.SchedulerEstimatorPort = 90000 + }), expectedErrs: field.ErrorList{field.Invalid(newPath.Child("SchedulerEstimatorPort"), 90000, "must be a valid port between 0 and 65535 inclusive")}, }, "invalid SchedulerEstimatorTimeout": { - opt: Options{ - LeaderElection: componentbaseconfig.LeaderElectionConfiguration{ - LeaderElect: false, - }, - BindAddress: "127.0.0.1", - SecurePort: 9000, - KubeAPIQPS: 40, - KubeAPIBurst: 30, - EnableSchedulerEstimator: false, - SchedulerEstimatorTimeout: metav1.Duration{Duration: -1 * time.Second}, - SchedulerEstimatorPort: 9000, - }, + opt: New(func(option *Options) { + option.SchedulerEstimatorTimeout = metav1.Duration{Duration: -1 * time.Second} + }), expectedErrs: field.ErrorList{field.Invalid(newPath.Child("SchedulerEstimatorTimeout"), metav1.Duration{Duration: -1 * time.Second}, "must be greater than or equal to 0")}, }, } diff --git a/cmd/scheduler/app/scheduler.go b/cmd/scheduler/app/scheduler.go index ebf5c4061..3f53e3e39 100644 --- a/cmd/scheduler/app/scheduler.go +++ b/cmd/scheduler/app/scheduler.go @@ -27,6 +27,7 @@ import ( "github.com/karmada-io/karmada/pkg/scheduler/framework/runtime" "github.com/karmada-io/karmada/pkg/sharedcli" "github.com/karmada-io/karmada/pkg/sharedcli/klogflag" + "github.com/karmada-io/karmada/pkg/sharedcli/profileflag" "github.com/karmada-io/karmada/pkg/version" "github.com/karmada-io/karmada/pkg/version/sharedcommand" ) @@ -90,6 +91,8 @@ func run(opts *options.Options, stopChan <-chan struct{}, registryOptions ...Opt klog.Infof("karmada-scheduler version: %s", version.Get()) go serveHealthzAndMetrics(net.JoinHostPort(opts.BindAddress, strconv.Itoa(opts.SecurePort))) + profileflag.ListenAndServe(opts.ProfileOpts) + restConfig, err := clientcmd.BuildConfigFromFlags(opts.Master, opts.KubeConfig) if err != nil { return fmt.Errorf("error building kubeconfig: %s", err.Error()) diff --git a/cmd/webhook/app/options/options.go b/cmd/webhook/app/options/options.go index 1ce626cad..d9d7ef351 100644 --- a/cmd/webhook/app/options/options.go +++ b/cmd/webhook/app/options/options.go @@ -2,6 +2,8 @@ package options import ( "github.com/spf13/pflag" + + "github.com/karmada-io/karmada/pkg/sharedcli/profileflag" ) const ( @@ -44,6 +46,8 @@ type Options struct { // for serving health probes // Defaults to ":8000". HealthProbeBindAddress string + + ProfileOpts profileflag.Options } // NewOptions builds an empty options. @@ -68,4 +72,6 @@ func (o *Options) AddFlags(flags *pflag.FlagSet) { flags.IntVar(&o.KubeAPIBurst, "kube-api-burst", 60, "Burst to use while talking with karmada-apiserver. Doesn't cover events and node heartbeat apis which rate limiting is controlled by a different set of flags.") flags.StringVar(&o.MetricsBindAddress, "metrics-bind-address", ":8080", "The TCP address that the controller should bind to for serving prometheus metrics(e.g. 127.0.0.1:8088, :8088)") flags.StringVar(&o.HealthProbeBindAddress, "health-probe-bind-address", ":8000", "The TCP address that the controller should bind to for serving health probes(e.g. 127.0.0.1:8000, :8000)") + + o.ProfileOpts.AddFlags(flags) } diff --git a/cmd/webhook/app/options/validation_test.go b/cmd/webhook/app/options/validation_test.go index ab40d5126..e979ec035 100644 --- a/cmd/webhook/app/options/validation_test.go +++ b/cmd/webhook/app/options/validation_test.go @@ -6,14 +6,27 @@ import ( "k8s.io/apimachinery/pkg/util/validation/field" ) +// a callback function to modify options +type ModifyOptions func(option *Options) + +// New an Options with default parameters +func New(modifyOptions ModifyOptions) Options { + option := Options{ + BindAddress: "127.0.0.1", + SecurePort: 9000, + KubeAPIQPS: 40, + KubeAPIBurst: 30, + } + + if modifyOptions != nil { + modifyOptions(&option) + } + return option +} + func TestValidateKarmadaWebhookConfiguration(t *testing.T) { successCases := []Options{ - { - BindAddress: "127.0.0.1", - SecurePort: 9000, - KubeAPIQPS: 40, - KubeAPIBurst: 30, - }, + New(nil), } for _, successCases := range successCases { if errs := successCases.Validate(); len(errs) != 0 { @@ -26,21 +39,15 @@ func TestValidateKarmadaWebhookConfiguration(t *testing.T) { expectedErrs field.ErrorList }{ "invalid BindAddress": { - opt: Options{ - BindAddress: "127.0.0.1:8080", - SecurePort: 9000, - KubeAPIQPS: 40, - KubeAPIBurst: 30, - }, + opt: New(func(option *Options) { + option.BindAddress = "127.0.0.1:8080" + }), expectedErrs: field.ErrorList{field.Invalid(newPath.Child("BindAddress"), "127.0.0.1:8080", "not a valid textual representation of an IP address")}, }, "invalid SecurePort": { - opt: Options{ - BindAddress: "127.0.0.1", - SecurePort: 900000, - KubeAPIQPS: 40, - KubeAPIBurst: 30, - }, + opt: New(func(option *Options) { + option.SecurePort = 900000 + }), expectedErrs: field.ErrorList{field.Invalid(newPath.Child("SecurePort"), 900000, "must be a valid port between 0 and 65535 inclusive")}, }, } diff --git a/cmd/webhook/app/webhook.go b/cmd/webhook/app/webhook.go index 92721ba49..b1b56e92c 100644 --- a/cmd/webhook/app/webhook.go +++ b/cmd/webhook/app/webhook.go @@ -18,6 +18,7 @@ import ( "github.com/karmada-io/karmada/cmd/webhook/app/options" "github.com/karmada-io/karmada/pkg/sharedcli" "github.com/karmada-io/karmada/pkg/sharedcli/klogflag" + "github.com/karmada-io/karmada/pkg/sharedcli/profileflag" "github.com/karmada-io/karmada/pkg/util/gclient" "github.com/karmada-io/karmada/pkg/version" "github.com/karmada-io/karmada/pkg/version/sharedcommand" @@ -79,6 +80,9 @@ func NewWebhookCommand(ctx context.Context) *cobra.Command { // Run runs the webhook server with options. This should never exit. func Run(ctx context.Context, opts *options.Options) error { klog.Infof("karmada-webhook version: %s", version.Get()) + + profileflag.ListenAndServe(opts.ProfileOpts) + config, err := controllerruntime.GetConfig() if err != nil { panic(err) diff --git a/pkg/sharedcli/profileflag/profileflag.go b/pkg/sharedcli/profileflag/profileflag.go new file mode 100644 index 000000000..da2eb7fd8 --- /dev/null +++ b/pkg/sharedcli/profileflag/profileflag.go @@ -0,0 +1,48 @@ +package profileflag + +import ( + "net/http" + "net/http/pprof" + "os" + + "github.com/spf13/pflag" + "k8s.io/klog/v2" +) + +// Options are options for pprof. +type Options struct { + // EnableProfile is the flag about whether to enable pprof profiling. + EnableProfile bool + // ProfilePort is the TCP address for pprof profiling. + // Defaults to 127.0.0.1:6060 if unspecified. + ProfilingBindAddress string +} + +// AddFlags adds flags to the specified FlagSet. +func (o *Options) AddFlags(fs *pflag.FlagSet) { + fs.BoolVar(&o.EnableProfile, "enable-pprof", false, "Enable profiling via web interface host:port/debug/pprof/.") + fs.StringVar(&o.ProfilingBindAddress, "profiling-bind-address", ":6060", "The TCP address for serving profiling(e.g. 127.0.0.1:6060, :6060). This is only applicable if profiling is enabled.") +} + +func installHandlerForPProf(mux *http.ServeMux) { + mux.HandleFunc("/debug/pprof/", pprof.Index) + mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) + mux.HandleFunc("/debug/pprof/profile", pprof.Profile) + mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol) + mux.HandleFunc("/debug/pprof/trace", pprof.Trace) +} + +// ListenAndServe start a http server to enable pprof. +func ListenAndServe(opts Options) { + if opts.EnableProfile { + mux := http.NewServeMux() + installHandlerForPProf(mux) + klog.Infof("Starting profiling on port %s", opts.ProfilingBindAddress) + go func() { + if err := http.ListenAndServe(opts.ProfilingBindAddress, mux); err != nil { + klog.Errorf("Failed to enable profiling: %v", err) + os.Exit(1) + } + }() + } +}