From a3c50e7e866f4e84659fb686cd683ac8a0a8235c Mon Sep 17 00:00:00 2001 From: zhangyukun <38148677+jameszhangyukun@users.noreply.github.com> Date: Tue, 1 Nov 2022 15:10:22 +0800 Subject: [PATCH] Resource Interpreter Framework cache and reorganize configurations Signed-off-by: zhangyukun <38148677+jameszhangyukun@users.noreply.github.com> --- .../configmanager/accessor.go | 94 +++++++++++ .../configmanager/manager.go | 94 +++++++++++ .../configurableinterpreter/configurable.go | 157 ++++++++++++++++++ pkg/resourceinterpreter/interpreter.go | 7 +- 4 files changed, 350 insertions(+), 2 deletions(-) create mode 100644 pkg/resourceinterpreter/configurableinterpreter/configmanager/accessor.go create mode 100644 pkg/resourceinterpreter/configurableinterpreter/configmanager/manager.go create mode 100644 pkg/resourceinterpreter/configurableinterpreter/configurable.go diff --git a/pkg/resourceinterpreter/configurableinterpreter/configmanager/accessor.go b/pkg/resourceinterpreter/configurableinterpreter/configmanager/accessor.go new file mode 100644 index 000000000..9afa0135d --- /dev/null +++ b/pkg/resourceinterpreter/configurableinterpreter/configmanager/accessor.go @@ -0,0 +1,94 @@ +package configmanager + +import ( + "k8s.io/apimachinery/pkg/runtime/schema" + + configv1alpha1 "github.com/karmada-io/karmada/pkg/apis/config/v1alpha1" +) + +// CustomConfiguration provides base information about custom interpreter configuration +type CustomConfiguration interface { + Name() string + TargetResource() schema.GroupVersionKind +} + +// LuaScriptAccessor provides a common interface to get custom interpreter lua script +type LuaScriptAccessor interface { + CustomConfiguration + + GetRetentionLuaScript() string + GetReplicaResourceLuaScript() string + GetReplicaRevisionLuaScript() string + GetStatusReflectionLuaScript() string + GetStatusAggregationLuaScript() string + GetHealthInterpretationLuaScript() string + GetDependencyInterpretationLuaScript() string +} + +// CustomAccessor provides a common interface to get custom interpreter configuration. +type CustomAccessor interface { + LuaScriptAccessor +} + +type resourceCustomAccessor struct { + retention *configv1alpha1.LocalValueRetention + replicaResource *configv1alpha1.ReplicaResourceRequirement + replicaRevision *configv1alpha1.ReplicaRevision + statusReflection *configv1alpha1.StatusReflection + statusAggregation *configv1alpha1.StatusAggregation + healthInterpretation *configv1alpha1.HealthInterpretation + dependencyInterpretation *configv1alpha1.DependencyInterpretation + configurationName string + configurationTargetGVK schema.GroupVersionKind +} + +// NewResourceCustomAccessorAccessor creates an accessor for resource interpreter customization +func NewResourceCustomAccessorAccessor(customization *configv1alpha1.ResourceInterpreterCustomization) CustomAccessor { + return &resourceCustomAccessor{ + retention: customization.Spec.Customizations.Retention, + replicaResource: customization.Spec.Customizations.ReplicaResource, + replicaRevision: customization.Spec.Customizations.ReplicaRevision, + statusReflection: customization.Spec.Customizations.StatusReflection, + statusAggregation: customization.Spec.Customizations.StatusAggregation, + healthInterpretation: customization.Spec.Customizations.HealthInterpretation, + dependencyInterpretation: customization.Spec.Customizations.DependencyInterpretation, + configurationName: customization.Name, + configurationTargetGVK: schema.FromAPIVersionAndKind(customization.Spec.Target.APIVersion, customization.Spec.Target.Kind), + } +} + +func (a *resourceCustomAccessor) GetRetentionLuaScript() string { + return a.retention.LuaScript +} + +func (a *resourceCustomAccessor) GetReplicaResourceLuaScript() string { + return a.replicaRevision.LuaScript +} + +func (a *resourceCustomAccessor) GetReplicaRevisionLuaScript() string { + return a.replicaResource.LuaScript +} + +func (a *resourceCustomAccessor) GetStatusReflectionLuaScript() string { + return a.statusReflection.LuaScript +} + +func (a *resourceCustomAccessor) GetStatusAggregationLuaScript() string { + return a.statusAggregation.LuaScript +} + +func (a *resourceCustomAccessor) GetHealthInterpretationLuaScript() string { + return a.healthInterpretation.LuaScript +} + +func (a *resourceCustomAccessor) GetDependencyInterpretationLuaScript() string { + return a.healthInterpretation.LuaScript +} + +func (a *resourceCustomAccessor) Name() string { + return a.configurationName +} + +func (a *resourceCustomAccessor) TargetResource() schema.GroupVersionKind { + return a.configurationTargetGVK +} diff --git a/pkg/resourceinterpreter/configurableinterpreter/configmanager/manager.go b/pkg/resourceinterpreter/configurableinterpreter/configmanager/manager.go new file mode 100644 index 000000000..b79a16170 --- /dev/null +++ b/pkg/resourceinterpreter/configurableinterpreter/configmanager/manager.go @@ -0,0 +1,94 @@ +package configmanager + +import ( + "fmt" + "sync/atomic" + + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime/schema" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/client-go/tools/cache" + "k8s.io/klog/v2" + + configv1alpha1 "github.com/karmada-io/karmada/pkg/apis/config/v1alpha1" + "github.com/karmada-io/karmada/pkg/util/fedinformer" + "github.com/karmada-io/karmada/pkg/util/fedinformer/genericmanager" + "github.com/karmada-io/karmada/pkg/util/helper" +) + +var resourceInterpreterCustomizationsGVR = schema.GroupVersionResource{ + Group: configv1alpha1.GroupVersion.Group, + Version: configv1alpha1.GroupVersion.Version, + Resource: "resourceinterpretercustomizations", +} + +// ConfigManager can list custom resource interpreter. +type ConfigManager interface { + LuaScriptAccessors() map[schema.GroupVersionKind]LuaScriptAccessor + HasSynced() bool +} + +// interpreterConfigManager collects the resource interpreter customization. +type interpreterConfigManager struct { + initialSynced *atomic.Value + lister cache.GenericLister + configuration *atomic.Value +} + +// LuaScriptAccessors returns all cached configurations. +func (configManager *interpreterConfigManager) LuaScriptAccessors() map[schema.GroupVersionKind]LuaScriptAccessor { + return configManager.configuration.Load().(map[schema.GroupVersionKind]LuaScriptAccessor) +} + +// HasSynced returns true when the cache is synced. +func (configManager *interpreterConfigManager) HasSynced() bool { + if configManager.initialSynced.Load().(bool) { + return true + } + + if configManager.HasSynced() { + configManager.initialSynced.Store(true) + return true + } + return false +} + +// NewInterpreterConfigManager watches ResourceInterpreterCustomization and organizes +// the configurations in the cache. +func NewInterpreterConfigManager(inform genericmanager.SingleClusterInformerManager) ConfigManager { + manager := &interpreterConfigManager{ + lister: inform.Lister(resourceInterpreterCustomizationsGVR), + initialSynced: &atomic.Value{}, + configuration: &atomic.Value{}, + } + manager.configuration.Store(make(map[schema.GroupVersionKind]LuaScriptAccessor)) + manager.initialSynced.Store(false) + configHandlers := fedinformer.NewHandlerOnEvents( + func(_ interface{}) { manager.updateConfiguration() }, + func(_, _ interface{}) { manager.updateConfiguration() }, + func(_ interface{}) { manager.updateConfiguration() }) + inform.ForResource(resourceInterpreterCustomizationsGVR, configHandlers) + return manager +} + +func (configManager *interpreterConfigManager) updateConfiguration() { + configurations, err := configManager.lister.List(labels.Everything()) + if err != nil { + utilruntime.HandleError(fmt.Errorf("error updating configuration: %v", err)) + return + } + configs := make(map[schema.GroupVersionKind]CustomAccessor, len(configurations)) + + for _, c := range configurations { + config := &configv1alpha1.ResourceInterpreterCustomization{} + if err = helper.ConvertToTypedObject(c, config); err != nil { + klog.Errorf("Failed to transform ResourceInterpreterCustomization: %w", err) + return + } + key := schema.FromAPIVersionAndKind(config.Spec.Target.APIVersion, config.Spec.Target.Kind) + configs[key] = NewResourceCustomAccessorAccessor(config) + } + + configManager.configuration.Store(configs) + configManager.initialSynced.Store(true) +} diff --git a/pkg/resourceinterpreter/configurableinterpreter/configurable.go b/pkg/resourceinterpreter/configurableinterpreter/configurable.go new file mode 100644 index 000000000..d89b177c2 --- /dev/null +++ b/pkg/resourceinterpreter/configurableinterpreter/configurable.go @@ -0,0 +1,157 @@ +package configurableinterpreter + +import ( + "fmt" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/klog/v2" + + configv1alpha1 "github.com/karmada-io/karmada/pkg/apis/config/v1alpha1" + workv1alpha2 "github.com/karmada-io/karmada/pkg/apis/work/v1alpha2" + "github.com/karmada-io/karmada/pkg/resourceinterpreter/configurableinterpreter/configmanager" + "github.com/karmada-io/karmada/pkg/util/fedinformer/genericmanager" +) + +// ConfigurableInterpreter interprets resources with resource interpreter customizations. +type ConfigurableInterpreter struct { + // configManager caches all ResourceInterpreterCustomizations. + configManager configmanager.ConfigManager +} + +// NewConfigurableInterpreter builds a new interpreter by registering the +// event handler to the provided informer instance. +func NewConfigurableInterpreter(informer genericmanager.SingleClusterInformerManager) *ConfigurableInterpreter { + return &ConfigurableInterpreter{ + configManager: configmanager.NewInterpreterConfigManager(informer), + } +} + +// HookEnabled tells if any hook exist for specific resource gvk and operation type. +func (c *ConfigurableInterpreter) HookEnabled(kind schema.GroupVersionKind, operationType configv1alpha1.InterpreterOperation) bool { + if !c.configManager.HasSynced() { + klog.Errorf("not yet ready to handle request") + return false + } + accessors, exist := c.configManager.LuaScriptAccessors()[kind] + if !exist { + return false + } + switch operationType { + case configv1alpha1.InterpreterOperationAggregateStatus: + return len(accessors.GetStatusAggregationLuaScript()) > 0 + case configv1alpha1.InterpreterOperationInterpretHealth: + return len(accessors.GetHealthInterpretationLuaScript()) > 0 + case configv1alpha1.InterpreterOperationInterpretDependency: + return len(accessors.GetDependencyInterpretationLuaScript()) > 0 + case configv1alpha1.InterpreterOperationInterpretReplica: + return len(accessors.GetReplicaResourceLuaScript()) > 0 + case configv1alpha1.InterpreterOperationInterpretStatus: + return len(accessors.GetStatusReflectionLuaScript()) > 0 + case configv1alpha1.InterpreterOperationRetain: + return len(accessors.GetRetentionLuaScript()) > 0 + case configv1alpha1.InterpreterOperationReviseReplica: + return len(accessors.GetReplicaRevisionLuaScript()) > 0 + } + return false +} + +// GetReplicas returns the desired replicas of the object as well as the requirements of each replica. +func (c *ConfigurableInterpreter) GetReplicas(object *unstructured.Unstructured) (int32, *workv1alpha2.ReplicaRequirements, error) { + klog.Infof("ConfigurableInterpreter Execute ReviseReplica") + customAccessor := c.configManager.LuaScriptAccessors()[object.GroupVersionKind()] + if len(customAccessor.GetReplicaResourceLuaScript()) == 0 { + return 0, nil, fmt.Errorf("customized interpreter operation GetReplicas for %q not found", object.GroupVersionKind()) + } + + luaScript := customAccessor.GetReplicaResourceLuaScript + klog.Infof("lua script %s", luaScript) + + return 0, nil, nil +} + +// ReviseReplica revises the replica of the given object. +func (c *ConfigurableInterpreter) ReviseReplica(object *unstructured.Unstructured, replica int64) (*unstructured.Unstructured, error) { + klog.Infof("ConfigurableInterpreter Execute ReviseReplica") + customAccessor := c.configManager.LuaScriptAccessors()[object.GroupVersionKind()] + if len(customAccessor.GetReplicaRevisionLuaScript()) == 0 { + return nil, fmt.Errorf("customized interpreter operation ReviseReplica for %q not found", object.GroupVersionKind()) + } + + luaScript := customAccessor.GetReplicaRevisionLuaScript() + klog.Infof("lua script %s", luaScript) + + return nil, nil +} + +// Retain returns the objects that based on the "desired" object but with values retained from the "observed" object. +func (c *ConfigurableInterpreter) Retain(desired *unstructured.Unstructured, observed *unstructured.Unstructured) (retained *unstructured.Unstructured, err error) { + klog.Infof("ConfigurableInterpreter Execute Retain") + customAccessor := c.configManager.LuaScriptAccessors()[desired.GroupVersionKind()] + if len(customAccessor.GetRetentionLuaScript()) == 0 { + return nil, fmt.Errorf("customized interpreter operation Retain for %q not found", desired.GroupVersionKind()) + } + + luaScript := customAccessor.GetRetentionLuaScript() + klog.Infof("lua script %s", luaScript) + + return nil, err +} + +// AggregateStatus returns the objects that based on the 'object' but with status aggregated. +func (c *ConfigurableInterpreter) AggregateStatus(object *unstructured.Unstructured, aggregatedStatusItems []workv1alpha2.AggregatedStatusItem) (*unstructured.Unstructured, error) { + klog.Infof("ConfigurableInterpreter Execute AggregateStatus") + customAccessor := c.configManager.LuaScriptAccessors()[object.GroupVersionKind()] + if len(customAccessor.GetStatusAggregationLuaScript()) == 0 { + return nil, fmt.Errorf("customized interpreter AggregateStatus Retain for %q not found", object.GroupVersionKind()) + } + + luaScript := customAccessor.GetStatusAggregationLuaScript() + klog.Infof("lua script %s", luaScript) + + return nil, nil +} + +// GetDependencies returns the dependent resources of the given object. +func (c *ConfigurableInterpreter) GetDependencies(object *unstructured.Unstructured) (dependencies []configv1alpha1.DependentObjectReference, err error) { + klog.Infof("ConfigurableInterpreter Execute GetDependencies") + + customAccessor := c.configManager.LuaScriptAccessors()[object.GroupVersionKind()] + if len(customAccessor.GetDependencyInterpretationLuaScript()) == 0 { + return nil, fmt.Errorf("customized interpreter GetDependencies Retain for %q not found", object.GroupVersionKind()) + } + + luaScript := customAccessor.GetDependencyInterpretationLuaScript() + klog.Infof("lua script %s", luaScript) + + return nil, err +} + +// ReflectStatus returns the status of the object. +func (c *ConfigurableInterpreter) ReflectStatus(object *unstructured.Unstructured) (status *runtime.RawExtension, err error) { + klog.Infof("ConfigurableInterpreter Execute ReflectStatus") + customAccessor := c.configManager.LuaScriptAccessors()[object.GroupVersionKind()] + if len(customAccessor.GetStatusAggregationLuaScript()) == 0 { + return nil, fmt.Errorf("customized interpreter GetDependencies Retain for %q not found", object.GroupVersionKind()) + } + + luaScript := customAccessor.GetStatusAggregationLuaScript() + klog.Infof("lua script %s", luaScript) + + return nil, err +} + +// InterpretHealth returns the health state of the object. +func (c *ConfigurableInterpreter) InterpretHealth(object *unstructured.Unstructured) (bool, error) { + klog.Infof("ConfigurableInterpreter Execute InterpretHealth") + customAccessor := c.configManager.LuaScriptAccessors()[object.GroupVersionKind()] + if len(customAccessor.GetHealthInterpretationLuaScript()) == 0 { + return false, fmt.Errorf("customized interpreter GetHealthInterpretation for %q not found", object.GroupVersionKind()) + } + + luaScript := customAccessor.GetHealthInterpretationLuaScript() + klog.Infof("lua script %s", luaScript) + + return false, nil +} diff --git a/pkg/resourceinterpreter/interpreter.go b/pkg/resourceinterpreter/interpreter.go index 690f99813..cf7e449a9 100644 --- a/pkg/resourceinterpreter/interpreter.go +++ b/pkg/resourceinterpreter/interpreter.go @@ -10,6 +10,7 @@ import ( configv1alpha1 "github.com/karmada-io/karmada/pkg/apis/config/v1alpha1" workv1alpha2 "github.com/karmada-io/karmada/pkg/apis/work/v1alpha2" + "github.com/karmada-io/karmada/pkg/resourceinterpreter/configurableinterpreter" "github.com/karmada-io/karmada/pkg/resourceinterpreter/customizedinterpreter" "github.com/karmada-io/karmada/pkg/resourceinterpreter/customizedinterpreter/webhook" "github.com/karmada-io/karmada/pkg/resourceinterpreter/defaultinterpreter" @@ -58,8 +59,9 @@ func NewResourceInterpreter(informer genericmanager.SingleClusterInformerManager type customResourceInterpreterImpl struct { informer genericmanager.SingleClusterInformerManager - customizedInterpreter *customizedinterpreter.CustomizedInterpreter - defaultInterpreter *defaultinterpreter.DefaultInterpreter + customizedInterpreter *customizedinterpreter.CustomizedInterpreter + defaultInterpreter *defaultinterpreter.DefaultInterpreter + configurableInterpreter *configurableinterpreter.ConfigurableInterpreter } // Start starts running the component and will never stop running until the context is closed or an error occurs. @@ -70,6 +72,7 @@ func (i *customResourceInterpreterImpl) Start(ctx context.Context) (err error) { if err != nil { return } + i.configurableInterpreter = configurableinterpreter.NewConfigurableInterpreter(i.informer) i.defaultInterpreter = defaultinterpreter.NewDefaultInterpreter()