Merge pull request #2008 from Poor12/add-pprof

Add pprof for karmada components
This commit is contained in:
karmada-bot 2022-07-07 11:09:30 +08:00 committed by GitHub
commit 5ecb4ba1bb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
20 changed files with 271 additions and 231 deletions

View File

@ -34,6 +34,7 @@ import (
"github.com/karmada-io/karmada/pkg/resourceinterpreter"
"github.com/karmada-io/karmada/pkg/sharedcli"
"github.com/karmada-io/karmada/pkg/sharedcli/klogflag"
"github.com/karmada-io/karmada/pkg/sharedcli/profileflag"
"github.com/karmada-io/karmada/pkg/util"
"github.com/karmada-io/karmada/pkg/util/gclient"
"github.com/karmada-io/karmada/pkg/util/helper"
@ -104,6 +105,9 @@ func init() {
func run(ctx context.Context, karmadaConfig karmadactl.KarmadaConfig, opts *options.Options) error {
klog.Infof("karmada-agent version: %s", version.Get())
profileflag.ListenAndServe(opts.ProfileOpts)
controlPlaneRestConfig, err := karmadaConfig.GetRestConfig(opts.KarmadaContext, opts.KarmadaKubeConfig)
if err != nil {
return fmt.Errorf("error building kubeconfig of karmada control plane: %w", err)

View File

@ -10,6 +10,7 @@ import (
"k8s.io/client-go/tools/leaderelection/resourcelock"
componentbaseconfig "k8s.io/component-base/config"
"github.com/karmada-io/karmada/pkg/sharedcli/profileflag"
"github.com/karmada-io/karmada/pkg/sharedcli/ratelimiterflag"
"github.com/karmada-io/karmada/pkg/util"
)
@ -88,6 +89,7 @@ type Options struct {
MetricsBindAddress string
RateLimiterOpts ratelimiterflag.Options
ProfileOpts profileflag.Options
}
// NewOptions builds an default scheduler options.
@ -153,4 +155,5 @@ func (o *Options) AddFlags(fs *pflag.FlagSet, allControllers []string) {
fs.IntVar(&o.ConcurrentWorkSyncs, "concurrent-work-syncs", 5, "The number of Works that are allowed to sync concurrently.")
fs.StringVar(&o.MetricsBindAddress, "metrics-bind-address", ":8080", "The TCP address that the controller should bind to for serving prometheus metrics(e.g. 127.0.0.1:8088, :8088)")
o.RateLimiterOpts.AddFlags(fs)
o.ProfileOpts.AddFlags(fs)
}

View File

@ -10,7 +10,6 @@ import (
"github.com/spf13/pflag"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
utilerrors "k8s.io/apimachinery/pkg/util/errors"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apiserver/pkg/admission"
"k8s.io/apiserver/pkg/endpoints/openapi"
@ -21,6 +20,7 @@ import (
genericoptions "k8s.io/apiserver/pkg/server/options"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/client-go/kubernetes"
"k8s.io/klog/v2"
netutils "k8s.io/utils/net"
"github.com/karmada-io/karmada/pkg/aggregatedapiserver"
@ -28,7 +28,9 @@ import (
clientset "github.com/karmada-io/karmada/pkg/generated/clientset/versioned"
informers "github.com/karmada-io/karmada/pkg/generated/informers/externalversions"
generatedopenapi "github.com/karmada-io/karmada/pkg/generated/openapi"
"github.com/karmada-io/karmada/pkg/sharedcli/profileflag"
"github.com/karmada-io/karmada/pkg/util/lifted"
"github.com/karmada-io/karmada/pkg/version"
)
const defaultEtcdPathPrefix = "/registry"
@ -42,6 +44,8 @@ type Options struct {
KubeAPIQPS float32
// KubeAPIBurst is the burst to allow while talking with karmada-apiserver.
KubeAPIBurst int
ProfileOpts profileflag.Options
}
// NewOptions returns a new Options.
@ -63,6 +67,7 @@ func (o *Options) AddFlags(flags *pflag.FlagSet) {
flags.Float32Var(&o.KubeAPIQPS, "kube-api-qps", 40.0, "QPS to use while talking with karmada-apiserver. Doesn't cover events and node heartbeat apis which rate limiting is controlled by a different set of flags.")
flags.IntVar(&o.KubeAPIBurst, "kube-api-burst", 60, "Burst to use while talking with karmada-apiserver. Doesn't cover events and node heartbeat apis which rate limiting is controlled by a different set of flags.")
utilfeature.DefaultMutableFeatureGate.AddFlag(flags)
o.ProfileOpts.AddFlags(flags)
}
// Complete fills in fields required to have valid data.
@ -70,15 +75,12 @@ func (o *Options) Complete() error {
return nil
}
// Validate validates Options.
func (o *Options) Validate() error {
var errs []error
errs = append(errs, o.RecommendedOptions.Validate()...)
return utilerrors.NewAggregate(errs)
}
// Run runs the aggregated-apiserver with options. This should never exit.
func (o *Options) Run(ctx context.Context) error {
klog.Infof("karmada-aggregated-apiserver version: %s", version.Get())
profileflag.ListenAndServe(o.ProfileOpts)
config, err := o.Config()
if err != nil {
return err
@ -120,6 +122,7 @@ func (o *Options) Config() (*aggregatedapiserver.Config, error) {
o.SharedInformerFactory = informerFactory
return []admission.PluginInitializer{}, nil
}
o.RecommendedOptions.Features = &genericoptions.FeatureOptions{EnableProfiling: false}
serverConfig := genericapiserver.NewRecommendedConfig(aggregatedapiserver.Codecs)
serverConfig.LongRunningFunc = customLongRunningRequestCheck(sets.NewString("watch", "proxy"),

View File

@ -0,0 +1,13 @@
package options
import (
utilerrors "k8s.io/apimachinery/pkg/util/errors"
)
// Validate validates Options.
func (o *Options) Validate() error {
var errs []error
errs = append(errs, o.RecommendedOptions.Validate()...)
return utilerrors.NewAggregate(errs)
}

View File

@ -44,6 +44,7 @@ import (
"github.com/karmada-io/karmada/pkg/resourceinterpreter"
"github.com/karmada-io/karmada/pkg/sharedcli"
"github.com/karmada-io/karmada/pkg/sharedcli/klogflag"
"github.com/karmada-io/karmada/pkg/sharedcli/profileflag"
"github.com/karmada-io/karmada/pkg/util"
"github.com/karmada-io/karmada/pkg/util/gclient"
"github.com/karmada-io/karmada/pkg/util/helper"
@ -97,6 +98,9 @@ func NewControllerManagerCommand(ctx context.Context) *cobra.Command {
// Run runs the controller-manager with options. This should never exit.
func Run(ctx context.Context, opts *options.Options) error {
klog.Infof("karmada-controller-manager version: %s", version.Get())
profileflag.ListenAndServe(opts.ProfileOpts)
config, err := controllerruntime.GetConfig()
if err != nil {
panic(err)

View File

@ -11,6 +11,7 @@ import (
componentbaseconfig "k8s.io/component-base/config"
"github.com/karmada-io/karmada/pkg/features"
"github.com/karmada-io/karmada/pkg/sharedcli/profileflag"
"github.com/karmada-io/karmada/pkg/sharedcli/ratelimiterflag"
"github.com/karmada-io/karmada/pkg/util"
)
@ -112,6 +113,7 @@ type Options struct {
ConcurrentResourceTemplateSyncs int
RateLimiterOpts ratelimiterflag.Options
ProfileOpts profileflag.Options
}
// NewOptions builds an empty options.
@ -190,5 +192,6 @@ func (o *Options) AddFlags(flags *pflag.FlagSet, allControllers, disabledByDefau
flags.IntVar(&o.ConcurrentResourceTemplateSyncs, "concurrent-resource-template-syncs", 5, "The number of resource templates that are allowed to sync concurrently.")
o.RateLimiterOpts.AddFlags(flags)
o.ProfileOpts.AddFlags(flags)
features.FeatureGate.AddFlag(flags)
}

View File

@ -25,6 +25,7 @@ import (
karmadaclientset "github.com/karmada-io/karmada/pkg/generated/clientset/versioned"
"github.com/karmada-io/karmada/pkg/sharedcli"
"github.com/karmada-io/karmada/pkg/sharedcli/klogflag"
"github.com/karmada-io/karmada/pkg/sharedcli/profileflag"
"github.com/karmada-io/karmada/pkg/version"
"github.com/karmada-io/karmada/pkg/version/sharedcommand"
)
@ -81,6 +82,8 @@ func run(opts *options.Options, stopChan <-chan struct{}) error {
klog.Infof("Please make sure the karmada-scheduler-estimator of all member clusters has been deployed")
go serveHealthzAndMetrics(net.JoinHostPort(opts.BindAddress, strconv.Itoa(opts.SecurePort)))
profileflag.ListenAndServe(opts.ProfileOpts)
restConfig, err := clientcmd.BuildConfigFromFlags(opts.Master, opts.KubeConfig)
if err != nil {
return fmt.Errorf("error building kubeconfig: %s", err.Error())

View File

@ -8,6 +8,7 @@ import (
"k8s.io/client-go/tools/leaderelection/resourcelock"
componentbaseconfig "k8s.io/component-base/config"
"github.com/karmada-io/karmada/pkg/sharedcli/profileflag"
"github.com/karmada-io/karmada/pkg/util"
)
@ -47,6 +48,7 @@ type Options struct {
DeschedulingInterval metav1.Duration
// UnschedulableThreshold specifies the period of pod unschedulable condition.
UnschedulableThreshold metav1.Duration
ProfileOpts profileflag.Options
}
// NewOptions builds a default descheduler options.
@ -81,4 +83,5 @@ func (o *Options) AddFlags(fs *pflag.FlagSet) {
fs.IntVar(&o.SchedulerEstimatorPort, "scheduler-estimator-port", defaultEstimatorPort, "The secure port on which to connect the accurate scheduler estimator.")
fs.DurationVar(&o.DeschedulingInterval.Duration, "descheduling-interval", defaultDeschedulingInterval, "Time interval between two consecutive descheduler executions. Setting this value instructs the descheduler to run in a continuous loop at the interval specified.")
fs.DurationVar(&o.UnschedulableThreshold.Duration, "unschedulable-threshold", defaultUnschedulableThreshold, "The period of pod unschedulable condition. This value is considered as a classification standard of unschedulable replicas.")
o.ProfileOpts.AddFlags(fs)
}

View File

@ -9,34 +9,39 @@ import (
componentbaseconfig "k8s.io/component-base/config"
)
// a callback function to modify options
type ModifyOptions func(option *Options)
// New an Options with default parameters
func New(modifyOptions ModifyOptions) Options {
option := Options{
LeaderElection: componentbaseconfig.LeaderElectionConfiguration{
LeaderElect: false,
},
BindAddress: "127.0.0.1",
SecurePort: 9000,
KubeAPIQPS: 40,
KubeAPIBurst: 30,
SchedulerEstimatorTimeout: metav1.Duration{Duration: 1 * time.Second},
SchedulerEstimatorPort: 9001,
DeschedulingInterval: metav1.Duration{Duration: 1 * time.Second},
UnschedulableThreshold: metav1.Duration{Duration: 1 * time.Second},
}
if modifyOptions != nil {
modifyOptions(&option)
}
return option
}
func TestValidateKarmadaDescheduler(t *testing.T) {
successCases := []Options{
{
LeaderElection: componentbaseconfig.LeaderElectionConfiguration{
LeaderElect: false,
},
BindAddress: "127.0.0.1",
SecurePort: 9000,
KubeAPIQPS: 40,
KubeAPIBurst: 30,
SchedulerEstimatorTimeout: metav1.Duration{Duration: 1 * time.Second},
SchedulerEstimatorPort: 9001,
DeschedulingInterval: metav1.Duration{Duration: 1 * time.Second},
UnschedulableThreshold: metav1.Duration{Duration: 1 * time.Second},
},
{
LeaderElection: componentbaseconfig.LeaderElectionConfiguration{
New(nil),
New(func(option *Options) {
option.LeaderElection = componentbaseconfig.LeaderElectionConfiguration{
LeaderElect: true,
},
BindAddress: "127.0.0.1",
SecurePort: 9000,
KubeAPIQPS: 40,
KubeAPIBurst: 30,
SchedulerEstimatorTimeout: metav1.Duration{Duration: 1 * time.Second},
SchedulerEstimatorPort: 9001,
DeschedulingInterval: metav1.Duration{Duration: 1 * time.Second},
UnschedulableThreshold: metav1.Duration{Duration: 1 * time.Second},
}, {
}
}), {
LeaderElection: componentbaseconfig.LeaderElectionConfiguration{
LeaderElect: false,
},
@ -58,99 +63,39 @@ func TestValidateKarmadaDescheduler(t *testing.T) {
expectedErrs field.ErrorList
}{
"invalid BindAddress": {
opt: Options{
LeaderElection: componentbaseconfig.LeaderElectionConfiguration{
LeaderElect: false,
},
BindAddress: "127.0.0.1:8080",
SecurePort: 9000,
KubeAPIQPS: 40,
KubeAPIBurst: 30,
SchedulerEstimatorTimeout: metav1.Duration{Duration: 1 * time.Second},
SchedulerEstimatorPort: 9001,
DeschedulingInterval: metav1.Duration{Duration: 1 * time.Second},
UnschedulableThreshold: metav1.Duration{Duration: 1 * time.Second},
},
opt: New(func(option *Options) {
option.BindAddress = "127.0.0.1:8080"
}),
expectedErrs: field.ErrorList{field.Invalid(newPath.Child("BindAddress"), "127.0.0.1:8080", "not a valid textual representation of an IP address")},
},
"invalid SecurePort": {
opt: Options{
LeaderElection: componentbaseconfig.LeaderElectionConfiguration{
LeaderElect: false,
},
BindAddress: "127.0.0.1",
SecurePort: 90000,
KubeAPIQPS: 40,
KubeAPIBurst: 30,
SchedulerEstimatorTimeout: metav1.Duration{Duration: 1 * time.Second},
SchedulerEstimatorPort: 9001,
DeschedulingInterval: metav1.Duration{Duration: 1 * time.Second},
UnschedulableThreshold: metav1.Duration{Duration: 1 * time.Second},
},
opt: New(func(option *Options) {
option.SecurePort = 90000
}),
expectedErrs: field.ErrorList{field.Invalid(newPath.Child("SecurePort"), 90000, "must be a valid port between 0 and 65535 inclusive")},
},
"invalid SchedulerEstimatorPort": {
opt: Options{
LeaderElection: componentbaseconfig.LeaderElectionConfiguration{
LeaderElect: false,
},
BindAddress: "127.0.0.1",
SecurePort: 9000,
KubeAPIQPS: 40,
KubeAPIBurst: 30,
SchedulerEstimatorTimeout: metav1.Duration{Duration: 1 * time.Second},
SchedulerEstimatorPort: 90000,
DeschedulingInterval: metav1.Duration{Duration: 1 * time.Second},
UnschedulableThreshold: metav1.Duration{Duration: 1 * time.Second},
},
opt: New(func(option *Options) {
option.SchedulerEstimatorPort = 90000
}),
expectedErrs: field.ErrorList{field.Invalid(newPath.Child("SchedulerEstimatorPort"), 90000, "must be a valid port between 0 and 65535 inclusive")},
},
"invalid SchedulerEstimatorTimeout": {
opt: Options{
LeaderElection: componentbaseconfig.LeaderElectionConfiguration{
LeaderElect: false,
},
BindAddress: "127.0.0.1",
SecurePort: 9000,
KubeAPIQPS: 40,
KubeAPIBurst: 30,
SchedulerEstimatorTimeout: metav1.Duration{Duration: -1 * time.Second},
SchedulerEstimatorPort: 9000,
DeschedulingInterval: metav1.Duration{Duration: 1 * time.Second},
UnschedulableThreshold: metav1.Duration{Duration: 1 * time.Second},
},
opt: New(func(option *Options) {
option.SchedulerEstimatorTimeout = metav1.Duration{Duration: -1 * time.Second}
}),
expectedErrs: field.ErrorList{field.Invalid(newPath.Child("SchedulerEstimatorTimeout"), metav1.Duration{Duration: -1 * time.Second}, "must be greater than or equal to 0")},
},
"invalid DeschedulingInterval": {
opt: Options{
LeaderElection: componentbaseconfig.LeaderElectionConfiguration{
LeaderElect: false,
},
BindAddress: "127.0.0.1",
SecurePort: 9000,
KubeAPIQPS: 40,
KubeAPIBurst: 30,
SchedulerEstimatorTimeout: metav1.Duration{Duration: 1 * time.Second},
SchedulerEstimatorPort: 9000,
DeschedulingInterval: metav1.Duration{Duration: -1 * time.Second},
UnschedulableThreshold: metav1.Duration{Duration: 1 * time.Second},
},
opt: New(func(option *Options) {
option.DeschedulingInterval = metav1.Duration{Duration: -1 * time.Second}
}),
expectedErrs: field.ErrorList{field.Invalid(newPath.Child("DeschedulingInterval"), metav1.Duration{Duration: -1 * time.Second}, "must be greater than or equal to 0")},
},
"invalid UnschedulableThreshold": {
opt: Options{
LeaderElection: componentbaseconfig.LeaderElectionConfiguration{
LeaderElect: false,
},
BindAddress: "127.0.0.1",
SecurePort: 9000,
KubeAPIQPS: 40,
KubeAPIBurst: 30,
SchedulerEstimatorTimeout: metav1.Duration{Duration: 1 * time.Second},
SchedulerEstimatorPort: 9000,
DeschedulingInterval: metav1.Duration{Duration: 1 * time.Second},
UnschedulableThreshold: metav1.Duration{Duration: -1 * time.Second},
},
opt: New(func(option *Options) {
option.UnschedulableThreshold = metav1.Duration{Duration: -1 * time.Second}
}),
expectedErrs: field.ErrorList{field.Invalid(newPath.Child("UnschedulableThreshold"), metav1.Duration{Duration: -1 * time.Second}, "must be greater than or equal to 0")},
},
}

View File

@ -20,6 +20,7 @@ import (
searchv1alpha1 "github.com/karmada-io/karmada/pkg/apis/search/v1alpha1"
generatedopenapi "github.com/karmada-io/karmada/pkg/generated/openapi"
"github.com/karmada-io/karmada/pkg/search"
"github.com/karmada-io/karmada/pkg/sharedcli/profileflag"
"github.com/karmada-io/karmada/pkg/version"
)
@ -33,6 +34,8 @@ type Options struct {
KubeAPIQPS float32
// KubeAPIBurst is the burst to allow while talking with karmada-search.
KubeAPIBurst int
ProfileOpts profileflag.Options
}
// NewOptions returns a new Options.
@ -57,6 +60,7 @@ func (o *Options) AddFlags(flags *pflag.FlagSet) {
flags.IntVar(&o.KubeAPIBurst, "kube-api-burst", 60, "Burst to use while talking with karmada-apiserver. Doesn't cover events and node heartbeat apis which rate limiting is controlled by a different set of flags.")
utilfeature.DefaultMutableFeatureGate.AddFlag(flags)
o.ProfileOpts.AddFlags(flags)
}
// Complete fills in fields required to have valid data.
@ -68,6 +72,8 @@ func (o *Options) Complete() error {
func (o *Options) Run(ctx context.Context) error {
klog.Infof("karmada-search version: %s", version.Get())
profileflag.ListenAndServe(o.ProfileOpts)
config, err := o.Config()
if err != nil {
return err
@ -99,6 +105,8 @@ func (o *Options) Config() (*search.Config, error) {
return nil, fmt.Errorf("error creating self-signed certificates: %v", err)
}
o.RecommendedOptions.Features = &genericoptions.FeatureOptions{EnableProfiling: false}
serverConfig := genericapiserver.NewRecommendedConfig(searchscheme.Codecs)
serverConfig.OpenAPIConfig = genericapiserver.DefaultOpenAPIConfig(generatedopenapi.GetOpenAPIDefinitions, openapi.NewDefinitionNamer(searchscheme.Scheme))
serverConfig.OpenAPIConfig.Info.Title = "karmada-search"

View File

@ -2,6 +2,8 @@ package options
import (
"github.com/spf13/pflag"
"github.com/karmada-io/karmada/pkg/sharedcli/profileflag"
)
const (
@ -27,6 +29,7 @@ type Options struct {
ClusterAPIBurst int
// Parallelism defines the amount of parallelism in algorithms for estimating. Must be greater than 0. Defaults to 16.
Parallelism int
ProfileOpts profileflag.Options
}
// NewOptions builds an empty options.
@ -48,4 +51,5 @@ func (o *Options) AddFlags(fs *pflag.FlagSet) {
fs.Float32Var(&o.ClusterAPIQPS, "kube-api-qps", 20.0, "QPS to use while talking with apiserver. Doesn't cover events and node heartbeat apis which rate limiting is controlled by a different set of flags.")
fs.IntVar(&o.ClusterAPIBurst, "kube-api-burst", 30, "Burst to use while talking with apiserver. Doesn't cover events and node heartbeat apis which rate limiting is controlled by a different set of flags.")
fs.IntVar(&o.Parallelism, "parallelism", o.Parallelism, "Parallelism defines the amount of parallelism in algorithms for estimating. Must be greater than 0. Defaults to 16.")
o.ProfileOpts.AddFlags(fs)
}

View File

@ -6,14 +6,27 @@ import (
"k8s.io/apimachinery/pkg/util/validation/field"
)
// a callback function to modify options
type ModifyOptions func(option *Options)
// New an Options with default parameters
func New(modifyOptions ModifyOptions) Options {
option := Options{
ClusterName: "testCluster",
BindAddress: "0.0.0.0",
SecurePort: 10100,
ServerPort: 8088,
}
if modifyOptions != nil {
modifyOptions(&option)
}
return option
}
func TestValidateKarmadaSchedulerEstimator(t *testing.T) {
successCases := []Options{
{
ClusterName: "testCluster",
BindAddress: "0.0.0.0",
SecurePort: 10100,
ServerPort: 8088,
},
New(nil),
}
for _, successCase := range successCases {
if errs := successCase.Validate(); len(errs) != 0 {
@ -27,39 +40,27 @@ func TestValidateKarmadaSchedulerEstimator(t *testing.T) {
expectedErrs field.ErrorList
}{
"invalid ClusterName": {
opt: Options{
ClusterName: "",
BindAddress: "127.0.0.1",
SecurePort: 10100,
ServerPort: 8088,
},
opt: New(func(option *Options) {
option.ClusterName = ""
}),
expectedErrs: field.ErrorList{field.Invalid(newPath.Child("ClusterName"), "", "clusterName cannot be empty")},
},
"invalid BindAddress": {
opt: Options{
ClusterName: "testCluster",
BindAddress: "127.0.0.1:8082",
SecurePort: 10100,
ServerPort: 8088,
},
opt: New(func(option *Options) {
option.BindAddress = "127.0.0.1:8082"
}),
expectedErrs: field.ErrorList{field.Invalid(newPath.Child("BindAddress"), "127.0.0.1:8082", "not a valid textual representation of an IP address")},
},
"invalid SecurePort": {
opt: Options{
ClusterName: "testCluster",
BindAddress: "127.0.0.1",
SecurePort: 908188,
ServerPort: 8088,
},
opt: New(func(option *Options) {
option.SecurePort = 908188
}),
expectedErrs: field.ErrorList{field.Invalid(newPath.Child("SecurePort"), 908188, "must be a valid port between 0 and 65535 inclusive")},
},
"invalid ServerPort": {
opt: Options{
ClusterName: "testCluster",
BindAddress: "127.0.0.1",
SecurePort: 9089,
ServerPort: 80888,
},
opt: New(func(option *Options) {
option.ServerPort = 80888
}),
expectedErrs: field.ErrorList{field.Invalid(newPath.Child("ServerPort"), 80888, "must be a valid port between 0 and 65535 inclusive")},
},
}

View File

@ -21,6 +21,7 @@ import (
"github.com/karmada-io/karmada/pkg/estimator/server"
"github.com/karmada-io/karmada/pkg/sharedcli"
"github.com/karmada-io/karmada/pkg/sharedcli/klogflag"
"github.com/karmada-io/karmada/pkg/sharedcli/profileflag"
"github.com/karmada-io/karmada/pkg/version"
"github.com/karmada-io/karmada/pkg/version/sharedcommand"
)
@ -66,6 +67,8 @@ func run(ctx context.Context, opts *options.Options) error {
klog.Infof("karmada-scheduler-estimator version: %s", version.Get())
go serveHealthzAndMetrics(net.JoinHostPort(opts.BindAddress, strconv.Itoa(opts.SecurePort)))
profileflag.ListenAndServe(opts.ProfileOpts)
restConfig, err := clientcmd.BuildConfigFromFlags(opts.Master, opts.KubeConfig)
if err != nil {
return fmt.Errorf("error building kubeconfig: %s", err.Error())

View File

@ -9,6 +9,7 @@ import (
componentbaseconfig "k8s.io/component-base/config"
"github.com/karmada-io/karmada/pkg/features"
"github.com/karmada-io/karmada/pkg/sharedcli/profileflag"
"github.com/karmada-io/karmada/pkg/util"
)
@ -50,6 +51,7 @@ type Options struct {
// EnableEmptyWorkloadPropagation represents whether workload with 0 replicas could be propagated to member clusters.
EnableEmptyWorkloadPropagation bool
ProfileOpts profileflag.Options
}
// NewOptions builds an default scheduler options.
@ -87,4 +89,5 @@ func (o *Options) AddFlags(fs *pflag.FlagSet) {
fs.IntVar(&o.SchedulerEstimatorPort, "scheduler-estimator-port", defaultEstimatorPort, "The secure port on which to connect the accurate scheduler estimator.")
fs.BoolVar(&o.EnableEmptyWorkloadPropagation, "enable-empty-workload-propagation", false, "Enable workload with replicas 0 to be propagated to member clusters.")
features.FeatureGate.AddFlag(fs)
o.ProfileOpts.AddFlags(fs)
}

View File

@ -9,32 +9,39 @@ import (
componentbaseconfig "k8s.io/component-base/config"
)
// a callback function to modify options
type ModifyOptions func(option *Options)
// New an Options with default parameters
func New(modifyOptions ModifyOptions) Options {
option := Options{
LeaderElection: componentbaseconfig.LeaderElectionConfiguration{
LeaderElect: false,
},
BindAddress: "127.0.0.1",
SecurePort: 9000,
KubeAPIQPS: 40,
KubeAPIBurst: 30,
EnableSchedulerEstimator: false,
SchedulerEstimatorTimeout: metav1.Duration{Duration: 1 * time.Second},
SchedulerEstimatorPort: 9001,
}
if modifyOptions != nil {
modifyOptions(&option)
}
return option
}
func TestValidateKarmadaSchedulerConfiguration(t *testing.T) {
successCases := []Options{
{
LeaderElection: componentbaseconfig.LeaderElectionConfiguration{
LeaderElect: false,
},
BindAddress: "127.0.0.1",
SecurePort: 9000,
KubeAPIQPS: 40,
KubeAPIBurst: 30,
EnableSchedulerEstimator: false,
SchedulerEstimatorTimeout: metav1.Duration{Duration: 1 * time.Second},
SchedulerEstimatorPort: 9001,
},
{
LeaderElection: componentbaseconfig.LeaderElectionConfiguration{
New(nil),
New(func(option *Options) {
option.LeaderElection = componentbaseconfig.LeaderElectionConfiguration{
LeaderElect: true,
},
BindAddress: "127.0.0.1",
SecurePort: 9000,
KubeAPIQPS: 40,
KubeAPIBurst: 30,
EnableSchedulerEstimator: false,
SchedulerEstimatorTimeout: metav1.Duration{Duration: 1 * time.Second},
SchedulerEstimatorPort: 9001,
}, {
}
}),
{
LeaderElection: componentbaseconfig.LeaderElectionConfiguration{
LeaderElect: false,
},
@ -42,7 +49,8 @@ func TestValidateKarmadaSchedulerConfiguration(t *testing.T) {
SecurePort: 9000,
KubeAPIQPS: 40,
KubeAPIBurst: 30,
}}
},
}
for _, successCase := range successCases {
if errs := successCase.Validate(); len(errs) != 0 {
@ -56,63 +64,27 @@ func TestValidateKarmadaSchedulerConfiguration(t *testing.T) {
expectedErrs field.ErrorList
}{
"invalid BindAddress": {
opt: Options{
LeaderElection: componentbaseconfig.LeaderElectionConfiguration{
LeaderElect: false,
},
BindAddress: "127.0.0.1:8080",
SecurePort: 9000,
KubeAPIQPS: 40,
KubeAPIBurst: 30,
EnableSchedulerEstimator: false,
SchedulerEstimatorTimeout: metav1.Duration{Duration: 1 * time.Second},
SchedulerEstimatorPort: 9001,
},
opt: New(func(option *Options) {
option.BindAddress = "127.0.0.1:8080"
}),
expectedErrs: field.ErrorList{field.Invalid(newPath.Child("BindAddress"), "127.0.0.1:8080", "not a valid textual representation of an IP address")},
},
"invalid SecurePort": {
opt: Options{
LeaderElection: componentbaseconfig.LeaderElectionConfiguration{
LeaderElect: false,
},
BindAddress: "127.0.0.1",
SecurePort: 90000,
KubeAPIQPS: 40,
KubeAPIBurst: 30,
EnableSchedulerEstimator: false,
SchedulerEstimatorTimeout: metav1.Duration{Duration: 1 * time.Second},
SchedulerEstimatorPort: 9001,
},
opt: New(func(option *Options) {
option.SecurePort = 90000
}),
expectedErrs: field.ErrorList{field.Invalid(newPath.Child("SecurePort"), 90000, "must be a valid port between 0 and 65535 inclusive")},
},
"invalid SchedulerEstimatorPort": {
opt: Options{
LeaderElection: componentbaseconfig.LeaderElectionConfiguration{
LeaderElect: false,
},
BindAddress: "127.0.0.1",
SecurePort: 9000,
KubeAPIQPS: 40,
KubeAPIBurst: 30,
EnableSchedulerEstimator: false,
SchedulerEstimatorTimeout: metav1.Duration{Duration: 1 * time.Second},
SchedulerEstimatorPort: 90000,
},
opt: New(func(option *Options) {
option.SchedulerEstimatorPort = 90000
}),
expectedErrs: field.ErrorList{field.Invalid(newPath.Child("SchedulerEstimatorPort"), 90000, "must be a valid port between 0 and 65535 inclusive")},
},
"invalid SchedulerEstimatorTimeout": {
opt: Options{
LeaderElection: componentbaseconfig.LeaderElectionConfiguration{
LeaderElect: false,
},
BindAddress: "127.0.0.1",
SecurePort: 9000,
KubeAPIQPS: 40,
KubeAPIBurst: 30,
EnableSchedulerEstimator: false,
SchedulerEstimatorTimeout: metav1.Duration{Duration: -1 * time.Second},
SchedulerEstimatorPort: 9000,
},
opt: New(func(option *Options) {
option.SchedulerEstimatorTimeout = metav1.Duration{Duration: -1 * time.Second}
}),
expectedErrs: field.ErrorList{field.Invalid(newPath.Child("SchedulerEstimatorTimeout"), metav1.Duration{Duration: -1 * time.Second}, "must be greater than or equal to 0")},
},
}

View File

@ -27,6 +27,7 @@ import (
"github.com/karmada-io/karmada/pkg/scheduler/framework/runtime"
"github.com/karmada-io/karmada/pkg/sharedcli"
"github.com/karmada-io/karmada/pkg/sharedcli/klogflag"
"github.com/karmada-io/karmada/pkg/sharedcli/profileflag"
"github.com/karmada-io/karmada/pkg/version"
"github.com/karmada-io/karmada/pkg/version/sharedcommand"
)
@ -90,6 +91,8 @@ func run(opts *options.Options, stopChan <-chan struct{}, registryOptions ...Opt
klog.Infof("karmada-scheduler version: %s", version.Get())
go serveHealthzAndMetrics(net.JoinHostPort(opts.BindAddress, strconv.Itoa(opts.SecurePort)))
profileflag.ListenAndServe(opts.ProfileOpts)
restConfig, err := clientcmd.BuildConfigFromFlags(opts.Master, opts.KubeConfig)
if err != nil {
return fmt.Errorf("error building kubeconfig: %s", err.Error())

View File

@ -2,6 +2,8 @@ package options
import (
"github.com/spf13/pflag"
"github.com/karmada-io/karmada/pkg/sharedcli/profileflag"
)
const (
@ -44,6 +46,8 @@ type Options struct {
// for serving health probes
// Defaults to ":8000".
HealthProbeBindAddress string
ProfileOpts profileflag.Options
}
// NewOptions builds an empty options.
@ -68,4 +72,6 @@ func (o *Options) AddFlags(flags *pflag.FlagSet) {
flags.IntVar(&o.KubeAPIBurst, "kube-api-burst", 60, "Burst to use while talking with karmada-apiserver. Doesn't cover events and node heartbeat apis which rate limiting is controlled by a different set of flags.")
flags.StringVar(&o.MetricsBindAddress, "metrics-bind-address", ":8080", "The TCP address that the controller should bind to for serving prometheus metrics(e.g. 127.0.0.1:8088, :8088)")
flags.StringVar(&o.HealthProbeBindAddress, "health-probe-bind-address", ":8000", "The TCP address that the controller should bind to for serving health probes(e.g. 127.0.0.1:8000, :8000)")
o.ProfileOpts.AddFlags(flags)
}

View File

@ -6,14 +6,27 @@ import (
"k8s.io/apimachinery/pkg/util/validation/field"
)
// a callback function to modify options
type ModifyOptions func(option *Options)
// New an Options with default parameters
func New(modifyOptions ModifyOptions) Options {
option := Options{
BindAddress: "127.0.0.1",
SecurePort: 9000,
KubeAPIQPS: 40,
KubeAPIBurst: 30,
}
if modifyOptions != nil {
modifyOptions(&option)
}
return option
}
func TestValidateKarmadaWebhookConfiguration(t *testing.T) {
successCases := []Options{
{
BindAddress: "127.0.0.1",
SecurePort: 9000,
KubeAPIQPS: 40,
KubeAPIBurst: 30,
},
New(nil),
}
for _, successCases := range successCases {
if errs := successCases.Validate(); len(errs) != 0 {
@ -26,21 +39,15 @@ func TestValidateKarmadaWebhookConfiguration(t *testing.T) {
expectedErrs field.ErrorList
}{
"invalid BindAddress": {
opt: Options{
BindAddress: "127.0.0.1:8080",
SecurePort: 9000,
KubeAPIQPS: 40,
KubeAPIBurst: 30,
},
opt: New(func(option *Options) {
option.BindAddress = "127.0.0.1:8080"
}),
expectedErrs: field.ErrorList{field.Invalid(newPath.Child("BindAddress"), "127.0.0.1:8080", "not a valid textual representation of an IP address")},
},
"invalid SecurePort": {
opt: Options{
BindAddress: "127.0.0.1",
SecurePort: 900000,
KubeAPIQPS: 40,
KubeAPIBurst: 30,
},
opt: New(func(option *Options) {
option.SecurePort = 900000
}),
expectedErrs: field.ErrorList{field.Invalid(newPath.Child("SecurePort"), 900000, "must be a valid port between 0 and 65535 inclusive")},
},
}

View File

@ -18,6 +18,7 @@ import (
"github.com/karmada-io/karmada/cmd/webhook/app/options"
"github.com/karmada-io/karmada/pkg/sharedcli"
"github.com/karmada-io/karmada/pkg/sharedcli/klogflag"
"github.com/karmada-io/karmada/pkg/sharedcli/profileflag"
"github.com/karmada-io/karmada/pkg/util/gclient"
"github.com/karmada-io/karmada/pkg/version"
"github.com/karmada-io/karmada/pkg/version/sharedcommand"
@ -79,6 +80,9 @@ func NewWebhookCommand(ctx context.Context) *cobra.Command {
// Run runs the webhook server with options. This should never exit.
func Run(ctx context.Context, opts *options.Options) error {
klog.Infof("karmada-webhook version: %s", version.Get())
profileflag.ListenAndServe(opts.ProfileOpts)
config, err := controllerruntime.GetConfig()
if err != nil {
panic(err)

View File

@ -0,0 +1,48 @@
package profileflag
import (
"net/http"
"net/http/pprof"
"os"
"github.com/spf13/pflag"
"k8s.io/klog/v2"
)
// Options are options for pprof.
type Options struct {
// EnableProfile is the flag about whether to enable pprof profiling.
EnableProfile bool
// ProfilePort is the TCP address for pprof profiling.
// Defaults to 127.0.0.1:6060 if unspecified.
ProfilingBindAddress string
}
// AddFlags adds flags to the specified FlagSet.
func (o *Options) AddFlags(fs *pflag.FlagSet) {
fs.BoolVar(&o.EnableProfile, "enable-pprof", false, "Enable profiling via web interface host:port/debug/pprof/.")
fs.StringVar(&o.ProfilingBindAddress, "profiling-bind-address", ":6060", "The TCP address for serving profiling(e.g. 127.0.0.1:6060, :6060). This is only applicable if profiling is enabled.")
}
func installHandlerForPProf(mux *http.ServeMux) {
mux.HandleFunc("/debug/pprof/", pprof.Index)
mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
mux.HandleFunc("/debug/pprof/profile", pprof.Profile)
mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
mux.HandleFunc("/debug/pprof/trace", pprof.Trace)
}
// ListenAndServe start a http server to enable pprof.
func ListenAndServe(opts Options) {
if opts.EnableProfile {
mux := http.NewServeMux()
installHandlerForPProf(mux)
klog.Infof("Starting profiling on port %s", opts.ProfilingBindAddress)
go func() {
if err := http.ListenAndServe(opts.ProfilingBindAddress, mux); err != nil {
klog.Errorf("Failed to enable profiling: %v", err)
os.Exit(1)
}
}()
}
}