From cd9457dbfcef3a5e027c7ff594fe963d2f95c16b Mon Sep 17 00:00:00 2001 From: Ben Luddy Date: Mon, 26 Jun 2023 17:00:26 -0400 Subject: [PATCH] Deep disablement for APF based on --enable-priority-and-fairness. Avoids starting informers or the config-consuming controller when --enable-priority-and-fairness=false. For kube-apiserver, the config-producing controller runs if and only if flowcontrol API storage is enabled. Kubernetes-commit: 83f5b5c240e5cced1371bbd22e458dae43975238 --- pkg/server/options/feature.go | 25 +++++++++++++++++++- pkg/server/options/recommended.go | 30 ++++-------------------- pkg/server/options/server_run_options.go | 7 +----- 3 files changed, 30 insertions(+), 32 deletions(-) diff --git a/pkg/server/options/feature.go b/pkg/server/options/feature.go index 35596fba6..0f2ba3627 100644 --- a/pkg/server/options/feature.go +++ b/pkg/server/options/feature.go @@ -17,16 +17,24 @@ limitations under the License. package options import ( + "fmt" + "github.com/spf13/pflag" "k8s.io/apimachinery/pkg/runtime/serializer" + "k8s.io/apiserver/pkg/features" "k8s.io/apiserver/pkg/server" + "k8s.io/apiserver/pkg/util/feature" + utilflowcontrol "k8s.io/apiserver/pkg/util/flowcontrol" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes" ) type FeatureOptions struct { EnableProfiling bool DebugSocketPath string EnableContentionProfiling bool + EnablePriorityAndFairness bool } func NewFeatureOptions() *FeatureOptions { @@ -36,6 +44,7 @@ func NewFeatureOptions() *FeatureOptions { EnableProfiling: defaults.EnableProfiling, DebugSocketPath: defaults.DebugSocketPath, EnableContentionProfiling: defaults.EnableContentionProfiling, + EnablePriorityAndFairness: true, } } @@ -50,9 +59,11 @@ func (o *FeatureOptions) AddFlags(fs *pflag.FlagSet) { "Enable block profiling, if profiling is enabled") fs.StringVar(&o.DebugSocketPath, "debug-socket-path", o.DebugSocketPath, "Use an unprotected (no authn/authz) unix-domain socket for profiling with the given path") + fs.BoolVar(&o.EnablePriorityAndFairness, "enable-priority-and-fairness", o.EnablePriorityAndFairness, ""+ + "If true and the APIPriorityAndFairness feature gate is enabled, replace the max-in-flight handler with an enhanced one that queues and dispatches with priority and fairness") } -func (o *FeatureOptions) ApplyTo(c *server.Config) error { +func (o *FeatureOptions) ApplyTo(c *server.Config, clientset kubernetes.Interface, informers informers.SharedInformerFactory) error { if o == nil { return nil } @@ -61,6 +72,18 @@ func (o *FeatureOptions) ApplyTo(c *server.Config) error { c.DebugSocketPath = o.DebugSocketPath c.EnableContentionProfiling = o.EnableContentionProfiling + if o.EnablePriorityAndFairness && feature.DefaultFeatureGate.Enabled(features.APIPriorityAndFairness) { + if c.MaxRequestsInFlight+c.MaxMutatingRequestsInFlight <= 0 { + return fmt.Errorf("invalid configuration: MaxRequestsInFlight=%d and MaxMutatingRequestsInFlight=%d; they must add up to something positive", c.MaxRequestsInFlight, c.MaxMutatingRequestsInFlight) + + } + c.FlowControl = utilflowcontrol.New( + informers, + clientset.FlowcontrolV1beta3(), + c.MaxRequestsInFlight+c.MaxMutatingRequestsInFlight, + ) + } + return nil } diff --git a/pkg/server/options/recommended.go b/pkg/server/options/recommended.go index 5d031e202..eb7e67b36 100644 --- a/pkg/server/options/recommended.go +++ b/pkg/server/options/recommended.go @@ -17,20 +17,15 @@ limitations under the License. package options import ( - "fmt" - "github.com/spf13/pflag" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apiserver/pkg/admission" - "k8s.io/apiserver/pkg/features" "k8s.io/apiserver/pkg/server" "k8s.io/apiserver/pkg/storage/storagebackend" "k8s.io/apiserver/pkg/util/feature" - utilflowcontrol "k8s.io/apiserver/pkg/util/flowcontrol" "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" "k8s.io/component-base/featuregate" - "k8s.io/klog/v2" ) // RecommendedOptions contains the recommended options for running an API server. @@ -122,17 +117,17 @@ func (o *RecommendedOptions) ApplyTo(config *server.RecommendedConfig) error { if err := o.Audit.ApplyTo(&config.Config); err != nil { return err } - if err := o.Features.ApplyTo(&config.Config); err != nil { - return err - } if err := o.CoreAPI.ApplyTo(config); err != nil { return err } - initializers, err := o.ExtraAdmissionInitializers(config) + kubeClient, err := kubernetes.NewForConfig(config.ClientConfig) if err != nil { return err } - kubeClient, err := kubernetes.NewForConfig(config.ClientConfig) + if err := o.Features.ApplyTo(&config.Config, kubeClient, config.SharedInformerFactory); err != nil { + return err + } + initializers, err := o.ExtraAdmissionInitializers(config) if err != nil { return err } @@ -144,21 +139,6 @@ func (o *RecommendedOptions) ApplyTo(config *server.RecommendedConfig) error { initializers...); err != nil { return err } - if feature.DefaultFeatureGate.Enabled(features.APIPriorityAndFairness) { - if config.ClientConfig != nil { - if config.MaxRequestsInFlight+config.MaxMutatingRequestsInFlight <= 0 { - return fmt.Errorf("invalid configuration: MaxRequestsInFlight=%d and MaxMutatingRequestsInFlight=%d; they must add up to something positive", config.MaxRequestsInFlight, config.MaxMutatingRequestsInFlight) - - } - config.FlowControl = utilflowcontrol.New( - config.SharedInformerFactory, - kubernetes.NewForConfigOrDie(config.ClientConfig).FlowcontrolV1beta3(), - config.MaxRequestsInFlight+config.MaxMutatingRequestsInFlight, - ) - } else { - klog.Warningf("Neither kubeconfig is provided nor service-account is mounted, so APIPriorityAndFairness will be disabled") - } - } return nil } diff --git a/pkg/server/options/server_run_options.go b/pkg/server/options/server_run_options.go index f9d574d5d..1373d8a4d 100644 --- a/pkg/server/options/server_run_options.go +++ b/pkg/server/options/server_run_options.go @@ -62,8 +62,7 @@ type ServerRunOptions struct { // decoded in a write request. 0 means no limit. // We intentionally did not add a flag for this option. Users of the // apiserver library can wire it to a flag. - MaxRequestBodyBytes int64 - EnablePriorityAndFairness bool + MaxRequestBodyBytes int64 // ShutdownSendRetryAfter dictates when to initiate shutdown of the HTTP // Server during the graceful termination of the apiserver. If true, we wait @@ -104,7 +103,6 @@ func NewServerRunOptions() *ServerRunOptions { ShutdownWatchTerminationGracePeriod: defaults.ShutdownWatchTerminationGracePeriod, JSONPatchMaxCopyBytes: defaults.JSONPatchMaxCopyBytes, MaxRequestBodyBytes: defaults.MaxRequestBodyBytes, - EnablePriorityAndFairness: true, ShutdownSendRetryAfter: false, } } @@ -325,9 +323,6 @@ func (s *ServerRunOptions) AddUniversalFlags(fs *pflag.FlagSet) { "handler, which picks a randomized value above this number as the connection timeout, "+ "to spread out load.") - fs.BoolVar(&s.EnablePriorityAndFairness, "enable-priority-and-fairness", s.EnablePriorityAndFairness, ""+ - "If true and the APIPriorityAndFairness feature gate is enabled, replace the max-in-flight handler with an enhanced one that queues and dispatches with priority and fairness") - fs.DurationVar(&s.ShutdownDelayDuration, "shutdown-delay-duration", s.ShutdownDelayDuration, ""+ "Time to delay the termination. During that time the server keeps serving requests normally. The endpoints /healthz and /livez "+ "will return success, but /readyz immediately returns failure. Graceful termination starts after this delay "+