add feature gate for search and proxy

Signed-off-by: yingjinhui <yingjinhui@didiglobal.com>
This commit is contained in:
yingjinhui 2022-10-17 11:40:54 +08:00
parent 05be79f76e
commit b7554264ed
3 changed files with 60 additions and 56 deletions

View File

@ -47,6 +47,9 @@ type Options struct {
KubeAPIBurst int KubeAPIBurst int
ProfileOpts profileflag.Options ProfileOpts profileflag.Options
DisableSearch bool
DisableProxy bool
} }
// NewOptions returns a new Options. // NewOptions returns a new Options.
@ -69,6 +72,8 @@ func (o *Options) AddFlags(flags *pflag.FlagSet) {
flags.Float32Var(&o.KubeAPIQPS, "kube-api-qps", 40.0, "QPS to use while talking with karmada-apiserver. Doesn't cover events and node heartbeat apis which rate limiting is controlled by a different set of flags.") flags.Float32Var(&o.KubeAPIQPS, "kube-api-qps", 40.0, "QPS to use while talking with karmada-apiserver. Doesn't cover events and node heartbeat apis which rate limiting is controlled by a different set of flags.")
flags.IntVar(&o.KubeAPIBurst, "kube-api-burst", 60, "Burst to use while talking with karmada-apiserver. Doesn't cover events and node heartbeat apis which rate limiting is controlled by a different set of flags.") flags.IntVar(&o.KubeAPIBurst, "kube-api-burst", 60, "Burst to use while talking with karmada-apiserver. Doesn't cover events and node heartbeat apis which rate limiting is controlled by a different set of flags.")
flags.BoolVar(&o.DisableSearch, "disable-search", false, "Disable search feature that would save memory usage significantly.")
flags.BoolVar(&o.DisableProxy, "disable-proxy", false, "Disable proxy feature that would save memory usage significantly.")
utilfeature.DefaultMutableFeatureGate.AddFlag(flags) utilfeature.DefaultMutableFeatureGate.AddFlag(flags)
o.ProfileOpts.AddFlags(flags) o.ProfileOpts.AddFlags(flags)
@ -101,25 +106,29 @@ func (o *Options) Run(ctx context.Context) error {
}) })
server.GenericAPIServer.AddPostStartHookOrDie("start-karmada-informers", func(context genericapiserver.PostStartHookContext) error { server.GenericAPIServer.AddPostStartHookOrDie("start-karmada-informers", func(context genericapiserver.PostStartHookContext) error {
config.KarmadaSharedInformerFactory.Start(context.StopCh) config.ExtraConfig.KarmadaSharedInformerFactory.Start(context.StopCh)
return nil return nil
}) })
server.GenericAPIServer.AddPostStartHookOrDie("start-karmada-search-controller", func(context genericapiserver.PostStartHookContext) error { if config.ExtraConfig.Controller != nil {
// start ResourceRegistry controller server.GenericAPIServer.AddPostStartHookOrDie("start-karmada-search-controller", func(context genericapiserver.PostStartHookContext) error {
config.Controller.Start(context.StopCh) // start ResourceRegistry controller
return nil config.ExtraConfig.Controller.Start(context.StopCh)
}) return nil
})
}
server.GenericAPIServer.AddPostStartHookOrDie("start-karmada-proxy-controller", func(context genericapiserver.PostStartHookContext) error { if config.ExtraConfig.ProxyController != nil {
config.ProxyController.Start(context.StopCh) server.GenericAPIServer.AddPostStartHookOrDie("start-karmada-proxy-controller", func(context genericapiserver.PostStartHookContext) error {
return nil config.ExtraConfig.ProxyController.Start(context.StopCh)
}) return nil
})
server.GenericAPIServer.AddPreShutdownHookOrDie("stop-karmada-proxy-controller", func() error { server.GenericAPIServer.AddPreShutdownHookOrDie("stop-karmada-proxy-controller", func() error {
config.ProxyController.Stop() config.ExtraConfig.ProxyController.Stop()
return nil return nil
}) })
}
return server.GenericAPIServer.PrepareRun().Run(ctx.Done()) return server.GenericAPIServer.PrepareRun().Run(ctx.Done())
} }
@ -152,25 +161,33 @@ func (o *Options) Config() (*search.Config, error) {
return nil, err return nil, err
} }
ctl, err := search.NewController(serverConfig.ClientConfig)
if err != nil {
return nil, err
}
karmadaClient := karmadaclientset.NewForConfigOrDie(serverConfig.ClientConfig) karmadaClient := karmadaclientset.NewForConfigOrDie(serverConfig.ClientConfig)
factory := informerfactory.NewSharedInformerFactory(karmadaClient, 0) factory := informerfactory.NewSharedInformerFactory(karmadaClient, 0)
proxyCtl, err := proxy.NewController(serverConfig.ClientConfig, restMapper, serverConfig.SharedInformerFactory, factory, var ctl *search.Controller
time.Second*time.Duration(serverConfig.Config.MinRequestTimeout)) if !o.DisableSearch {
if err != nil { ctl, err = search.NewController(serverConfig.ClientConfig, factory, restMapper)
return nil, err if err != nil {
return nil, err
}
}
var proxyCtl *proxy.Controller
if !o.DisableProxy {
proxyCtl, err = proxy.NewController(serverConfig.ClientConfig, restMapper, serverConfig.SharedInformerFactory, factory,
time.Second*time.Duration(serverConfig.Config.MinRequestTimeout))
if err != nil {
return nil, err
}
} }
config := &search.Config{ config := &search.Config{
GenericConfig: serverConfig, GenericConfig: serverConfig,
Controller: ctl, ExtraConfig: search.ExtraConfig{
ProxyController: proxyCtl, KarmadaSharedInformerFactory: factory,
KarmadaSharedInformerFactory: factory, Controller: ctl,
ProxyController: proxyCtl,
},
} }
return config, nil return config, nil
} }

View File

@ -9,27 +9,22 @@ import (
searchapis "github.com/karmada-io/karmada/pkg/apis/search" searchapis "github.com/karmada-io/karmada/pkg/apis/search"
searchscheme "github.com/karmada-io/karmada/pkg/apis/search/scheme" searchscheme "github.com/karmada-io/karmada/pkg/apis/search/scheme"
informerfactory "github.com/karmada-io/karmada/pkg/generated/informers/externalversions" informerfactory "github.com/karmada-io/karmada/pkg/generated/informers/externalversions"
clusterlister "github.com/karmada-io/karmada/pkg/generated/listers/cluster/v1alpha1"
searchstorage "github.com/karmada-io/karmada/pkg/registry/search/storage" searchstorage "github.com/karmada-io/karmada/pkg/registry/search/storage"
"github.com/karmada-io/karmada/pkg/search/proxy" "github.com/karmada-io/karmada/pkg/search/proxy"
"github.com/karmada-io/karmada/pkg/util/fedinformer/genericmanager"
) )
// ExtraConfig holds custom apiserver config // ExtraConfig holds custom apiserver config
type ExtraConfig struct { type ExtraConfig struct {
MultiClusterInformerManager genericmanager.MultiClusterInformerManager
ClusterLister clusterlister.ClusterLister
KarmadaSharedInformerFactory informerfactory.SharedInformerFactory KarmadaSharedInformerFactory informerfactory.SharedInformerFactory
Controller *Controller
ProxyController *proxy.Controller ProxyController *proxy.Controller
// Add custom config if necessary. // Add custom config if necessary.
} }
// Config defines the config for the APIServer. // Config defines the config for the APIServer.
type Config struct { type Config struct {
GenericConfig *genericapiserver.RecommendedConfig GenericConfig *genericapiserver.RecommendedConfig
Controller *Controller ExtraConfig ExtraConfig
ProxyController *proxy.Controller
KarmadaSharedInformerFactory informerfactory.SharedInformerFactory
} }
// APIServer contains state for karmada-search. // APIServer contains state for karmada-search.
@ -51,12 +46,7 @@ type CompletedConfig struct {
func (cfg *Config) Complete() CompletedConfig { func (cfg *Config) Complete() CompletedConfig {
c := completedConfig{ c := completedConfig{
cfg.GenericConfig.Complete(), cfg.GenericConfig.Complete(),
&ExtraConfig{ &cfg.ExtraConfig,
MultiClusterInformerManager: cfg.Controller.InformerManager,
ClusterLister: cfg.Controller.clusterLister,
KarmadaSharedInformerFactory: cfg.KarmadaSharedInformerFactory,
ProxyController: cfg.ProxyController,
},
} }
c.GenericConfig.Version = &version.Info{ c.GenericConfig.Version = &version.Info{
@ -84,14 +74,21 @@ func (c completedConfig) New() (*APIServer, error) {
klog.Errorf("unable to create REST storage for a resource due to %v, will die", err) klog.Errorf("unable to create REST storage for a resource due to %v, will die", err)
return nil, err return nil, err
} }
searchREST := searchstorage.NewSearchREST(c.ExtraConfig.MultiClusterInformerManager, c.ExtraConfig.ClusterLister)
proxyingREST := searchstorage.NewProxyingREST(c.ExtraConfig.ProxyController)
v1alpha1search := map[string]rest.Storage{} v1alpha1search := map[string]rest.Storage{}
v1alpha1search["resourceregistries"] = resourceRegistryStorage.ResourceRegistry v1alpha1search["resourceregistries"] = resourceRegistryStorage.ResourceRegistry
v1alpha1search["resourceregistries/status"] = resourceRegistryStorage.Status v1alpha1search["resourceregistries/status"] = resourceRegistryStorage.Status
v1alpha1search["search"] = searchREST
v1alpha1search["proxying"] = proxyingREST if c.ExtraConfig.Controller != nil {
searchREST := searchstorage.NewSearchREST(c.ExtraConfig.Controller.InformerManager, c.ExtraConfig.Controller.clusterLister)
v1alpha1search["search"] = searchREST
}
if c.ExtraConfig.ProxyController != nil {
proxyingREST := searchstorage.NewProxyingREST(c.ExtraConfig.ProxyController)
v1alpha1search["proxying"] = proxyingREST
}
apiGroupInfo.VersionedResourcesStorageMap["v1alpha1"] = v1alpha1search apiGroupInfo.VersionedResourcesStorageMap["v1alpha1"] = v1alpha1search
if err = server.GenericAPIServer.InstallAPIGroup(&apiGroupInfo); err != nil { if err = server.GenericAPIServer.InstallAPIGroup(&apiGroupInfo); err != nil {

View File

@ -21,7 +21,6 @@ import (
clusterV1alpha1 "github.com/karmada-io/karmada/pkg/apis/cluster/v1alpha1" clusterV1alpha1 "github.com/karmada-io/karmada/pkg/apis/cluster/v1alpha1"
policyv1alpha1 "github.com/karmada-io/karmada/pkg/apis/policy/v1alpha1" policyv1alpha1 "github.com/karmada-io/karmada/pkg/apis/policy/v1alpha1"
searchv1alpha1 "github.com/karmada-io/karmada/pkg/apis/search/v1alpha1" searchv1alpha1 "github.com/karmada-io/karmada/pkg/apis/search/v1alpha1"
karmadaclientset "github.com/karmada-io/karmada/pkg/generated/clientset/versioned"
informerfactory "github.com/karmada-io/karmada/pkg/generated/informers/externalversions" informerfactory "github.com/karmada-io/karmada/pkg/generated/informers/externalversions"
clusterlister "github.com/karmada-io/karmada/pkg/generated/listers/cluster/v1alpha1" clusterlister "github.com/karmada-io/karmada/pkg/generated/listers/cluster/v1alpha1"
"github.com/karmada-io/karmada/pkg/search/backendstore" "github.com/karmada-io/karmada/pkg/search/backendstore"
@ -57,16 +56,8 @@ type Controller struct {
} }
// NewController returns a new ResourceRegistry controller // NewController returns a new ResourceRegistry controller
func NewController(restConfig *rest.Config) (*Controller, error) { func NewController(restConfig *rest.Config, factory informerfactory.SharedInformerFactory, restMapper meta.RESTMapper) (*Controller, error) {
karmadaClient := karmadaclientset.NewForConfigOrDie(restConfig)
factory := informerfactory.NewSharedInformerFactory(karmadaClient, 0)
clusterLister := factory.Cluster().V1alpha1().Clusters().Lister() clusterLister := factory.Cluster().V1alpha1().Clusters().Lister()
restMapper, err := restmapper.MapperProvider(restConfig)
if err != nil {
klog.Errorf("Failed to create REST mapper: %v", err)
return nil, err
}
queue := workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()) queue := workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter())
c := &Controller{ c := &Controller{
@ -115,7 +106,6 @@ func (c *Controller) Start(stopCh <-chan struct{}) {
defer runtime.HandleCrash() defer runtime.HandleCrash()
c.informerFactory.Start(stopCh)
c.informerFactory.WaitForCacheSync(stopCh) c.informerFactory.WaitForCacheSync(stopCh)
go wait.Until(c.worker, time.Second, stopCh) go wait.Until(c.worker, time.Second, stopCh)