diff --git a/pkg/resourceinterpreter/default/thirdparty/manager.go b/pkg/resourceinterpreter/default/thirdparty/manager.go new file mode 100644 index 000000000..6f34aa5f0 --- /dev/null +++ b/pkg/resourceinterpreter/default/thirdparty/manager.go @@ -0,0 +1,105 @@ +package thirdparty + +import ( + "bytes" + "io" + "io/fs" + "path/filepath" + "sort" + "strings" + "sync/atomic" + + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/yaml" + "k8s.io/klog/v2" + + configv1alpha1 "github.com/karmada-io/karmada/pkg/apis/config/v1alpha1" + "github.com/karmada-io/karmada/pkg/resourceinterpreter/customized/declarative/configmanager" + "github.com/karmada-io/karmada/pkg/resourceinterpreter/default/thirdparty/resourcecustomizations" +) + +const configurableInterpreterFile = "customizations.yaml" + +// configManager collects the thirdparty resource interpreter customization. +type configManager struct { + configuration atomic.Value +} + +func (t *configManager) HasSynced() bool { + return true +} + +func (t *configManager) LoadConfig(customizations []*configv1alpha1.ResourceInterpreterCustomization) { + sort.Slice(customizations, func(i, j int) bool { + return customizations[i].Name < customizations[j].Name + }) + + accessors := make(map[schema.GroupVersionKind]configmanager.CustomAccessor) + for _, config := range customizations { + key := schema.FromAPIVersionAndKind(config.Spec.Target.APIVersion, config.Spec.Target.Kind) + var ac configmanager.CustomAccessor + var ok bool + if ac, ok = accessors[key]; !ok { + ac = configmanager.NewResourceCustomAccessor() + } + ac.Merge(config.Spec.Customizations) + accessors[key] = ac + } + + t.configuration.Store(accessors) +} + +// CustomAccessors returns all cached configurations. +func (t *configManager) CustomAccessors() map[schema.GroupVersionKind]configmanager.CustomAccessor { + return t.configuration.Load().(map[schema.GroupVersionKind]configmanager.CustomAccessor) +} + +// NewThirdPartyConfigManager load third party resource in the cache. +func NewThirdPartyConfigManager() configmanager.ConfigManager { + manager := &configManager{} + manager.configuration.Store(make(map[schema.GroupVersionKind]configmanager.CustomAccessor)) + manager.loadThirdPartyConfig() + return manager +} + +func (t *configManager) loadThirdPartyConfig() { + var configs []*configv1alpha1.ResourceInterpreterCustomization + if err := fs.WalkDir(resourcecustomizations.Embedded, ".", func(path string, d fs.DirEntry, err error) error { + if err != nil { + // cannot happen + return err + } + if d.IsDir() { + return nil + } + if strings.Contains(d.Name(), "testdata") { + return nil + } + + if filepath.Base(d.Name()) != configurableInterpreterFile { + return nil + } + data, err := fs.ReadFile(resourcecustomizations.Embedded, path) + if err != nil { + // cannot happen + return err + } + decoder := yaml.NewYAMLOrJSONDecoder(bytes.NewReader(data), 4096) + for { + config := &configv1alpha1.ResourceInterpreterCustomization{} + err = decoder.Decode(config) + if err != nil { + break + } + configs = append(configs, config) + } + if err != io.EOF { + return err + } + return nil + }); err != nil { + klog.Warning(err, "failed to load third party resource") + return + } + t.LoadConfig(configs) +} diff --git a/pkg/resourceinterpreter/default/thirdparty/resourcecustomizations/embed.go b/pkg/resourceinterpreter/default/thirdparty/resourcecustomizations/embed.go new file mode 100644 index 000000000..d1502847f --- /dev/null +++ b/pkg/resourceinterpreter/default/thirdparty/resourcecustomizations/embed.go @@ -0,0 +1,10 @@ +package resourcecustomizations + +import ( + "embed" +) + +// Embedded contains embedded resource customization +// +//go:embed * +var Embedded embed.FS diff --git a/pkg/resourceinterpreter/default/thirdparty/thirdparty.go b/pkg/resourceinterpreter/default/thirdparty/thirdparty.go new file mode 100644 index 000000000..c2e39b512 --- /dev/null +++ b/pkg/resourceinterpreter/default/thirdparty/thirdparty.go @@ -0,0 +1,220 @@ +package thirdparty + +import ( + "sort" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/klog/v2" + + configv1alpha1 "github.com/karmada-io/karmada/pkg/apis/config/v1alpha1" + workv1alpha2 "github.com/karmada-io/karmada/pkg/apis/work/v1alpha2" + "github.com/karmada-io/karmada/pkg/resourceinterpreter/customized/declarative/configmanager" + "github.com/karmada-io/karmada/pkg/resourceinterpreter/customized/declarative/luavm" +) + +// ConfigurableInterpreter interprets resources with third party resource interpreter. +type ConfigurableInterpreter struct { + configManager configmanager.ConfigManager + luaVM *luavm.VM +} + +// HookEnabled tells if any hook exist for specific resource gvk and operation type. +func (p *ConfigurableInterpreter) HookEnabled(kind schema.GroupVersionKind, operationType configv1alpha1.InterpreterOperation) bool { + customAccessor, exist := p.getCustomAccessor(kind) + if !exist { + return exist + } + if operationType == configv1alpha1.InterpreterOperationInterpretDependency { + scripts := customAccessor.GetDependencyInterpretationLuaScripts() + return scripts != nil + } + var script string + switch operationType { + case configv1alpha1.InterpreterOperationAggregateStatus: + script = customAccessor.GetStatusAggregationLuaScript() + case configv1alpha1.InterpreterOperationInterpretHealth: + script = customAccessor.GetHealthInterpretationLuaScript() + case configv1alpha1.InterpreterOperationInterpretReplica: + script = customAccessor.GetReplicaResourceLuaScript() + case configv1alpha1.InterpreterOperationInterpretStatus: + script = customAccessor.GetStatusReflectionLuaScript() + case configv1alpha1.InterpreterOperationRetain: + script = customAccessor.GetRetentionLuaScript() + case configv1alpha1.InterpreterOperationReviseReplica: + script = customAccessor.GetReplicaRevisionLuaScript() + } + return len(script) > 0 +} + +// GetReplicas returns the desired replicas of the object as well as the requirements of each replica. +func (p *ConfigurableInterpreter) GetReplicas(object *unstructured.Unstructured) (replicas int32, requires *workv1alpha2.ReplicaRequirements, enabled bool, err error) { + klog.V(4).Infof("Get replicas for object: %v %s/%s with thirdparty configurable interpreter.", object.GroupVersionKind(), object.GetNamespace(), object.GetName()) + + customAccessor, enabled := p.getCustomAccessor(object.GroupVersionKind()) + if !enabled { + return + } + + script := customAccessor.GetReplicaResourceLuaScript() + if len(script) == 0 { + enabled = false + return + } + + replicas, requires, err = p.luaVM.GetReplicas(object, script) + return +} + +// ReviseReplica revises the replica of the given object. +func (p *ConfigurableInterpreter) ReviseReplica(object *unstructured.Unstructured, replica int64) (revised *unstructured.Unstructured, enabled bool, err error) { + klog.V(4).Infof("Revise replicas for object: %v %s/%s with thirdparty configurable interpreter.", object.GroupVersionKind(), object.GetNamespace(), object.GetName()) + + customAccessor, enabled := p.getCustomAccessor(object.GroupVersionKind()) + if !enabled { + return + } + + script := customAccessor.GetReplicaRevisionLuaScript() + if len(script) == 0 { + enabled = false + return + } + + revised, err = p.luaVM.ReviseReplica(object, replica, script) + return +} + +// Retain returns the objects that based on the "desired" object but with values retained from the "observed" object. +func (p *ConfigurableInterpreter) Retain(desired *unstructured.Unstructured, observed *unstructured.Unstructured) (retained *unstructured.Unstructured, enabled bool, err error) { + klog.V(4).Infof("Retain object: %v %s/%s with thirdparty configurable interpreter.", desired.GroupVersionKind(), desired.GetNamespace(), desired.GetName()) + + customAccessor, enabled := p.getCustomAccessor(desired.GroupVersionKind()) + if !enabled { + return + } + + script := customAccessor.GetRetentionLuaScript() + if len(script) == 0 { + enabled = false + return + } + + retained, err = p.luaVM.Retain(desired, observed, script) + return +} + +// AggregateStatus returns the objects that based on the 'object' but with status aggregated. +func (p *ConfigurableInterpreter) AggregateStatus(object *unstructured.Unstructured, aggregatedStatusItems []workv1alpha2.AggregatedStatusItem) (status *unstructured.Unstructured, enabled bool, err error) { + klog.V(4).Infof("Aggregate status of object: %v %s/%s with thirdparty configurable interpreter.", object.GroupVersionKind(), object.GetNamespace(), object.GetName()) + customAccessor, enabled := p.getCustomAccessor(object.GroupVersionKind()) + if !enabled { + return + } + + script := customAccessor.GetStatusAggregationLuaScript() + if len(script) == 0 { + enabled = false + return + } + + status, err = p.luaVM.AggregateStatus(object, aggregatedStatusItems, script) + return +} + +// GetDependencies returns the dependent resources of the given object. +func (p *ConfigurableInterpreter) GetDependencies(object *unstructured.Unstructured) (dependencies []configv1alpha1.DependentObjectReference, enabled bool, err error) { + klog.V(4).Infof("Get dependencies of object: %v %s/%s with thirdparty configurable interpreter.", object.GroupVersionKind(), object.GetNamespace(), object.GetName()) + + customAccessor, enabled := p.getCustomAccessor(object.GroupVersionKind()) + if !enabled { + return + } + + scripts := customAccessor.GetDependencyInterpretationLuaScripts() + if scripts == nil { + enabled = false + return + } + + refs := sets.New[configv1alpha1.DependentObjectReference]() + for _, luaScript := range scripts { + var references []configv1alpha1.DependentObjectReference + references, err = p.luaVM.GetDependencies(object, luaScript) + if err != nil { + klog.Errorf("Failed to get DependentObjectReferences from object: %v %s/%s, error: %v", + object.GroupVersionKind(), object.GetNamespace(), object.GetName(), err) + return + } + refs.Insert(references...) + } + dependencies = refs.UnsortedList() + + // keep returned items in the same order between each call. + sort.Slice(dependencies, func(i, j int) bool { + if dependencies[i].APIVersion != dependencies[j].APIVersion { + return dependencies[i].APIVersion < dependencies[j].APIVersion + } + if dependencies[i].Kind != dependencies[j].Kind { + return dependencies[i].Kind < dependencies[j].Kind + } + if dependencies[i].Namespace != dependencies[j].Namespace { + return dependencies[i].Namespace < dependencies[j].Namespace + } + return dependencies[i].Name < dependencies[j].Name + }) + return +} + +// ReflectStatus returns the status of the object. +func (p *ConfigurableInterpreter) ReflectStatus(object *unstructured.Unstructured) (status *runtime.RawExtension, enabled bool, err error) { + klog.V(4).Infof("Reflect status of object: %v %s/%s with thirdparty configurable interpreter.", object.GroupVersionKind(), object.GetNamespace(), object.GetName()) + + customAccessor, enabled := p.getCustomAccessor(object.GroupVersionKind()) + if !enabled { + return + } + + script := customAccessor.GetStatusReflectionLuaScript() + if len(script) == 0 { + enabled = false + return + } + + status, err = p.luaVM.ReflectStatus(object, script) + return +} + +// InterpretHealth returns the health state of the object. +func (p *ConfigurableInterpreter) InterpretHealth(object *unstructured.Unstructured) (health bool, enabled bool, err error) { + klog.V(4).Infof("Get health status of object: %v %s/%s with thirdparty configurable interpreter.", object.GroupVersionKind(), object.GetNamespace(), object.GetName()) + + customAccessor, enabled := p.getCustomAccessor(object.GroupVersionKind()) + if !enabled { + return + } + + script := customAccessor.GetHealthInterpretationLuaScript() + if len(script) == 0 { + enabled = false + return + } + + health, err = p.luaVM.InterpretHealth(object, script) + return +} + +func (p *ConfigurableInterpreter) getCustomAccessor(kind schema.GroupVersionKind) (configmanager.CustomAccessor, bool) { + customAccessor, exist := p.configManager.CustomAccessors()[kind] + return customAccessor, exist +} + +// NewConfigurableInterpreter return a new ConfigurableInterpreter. +func NewConfigurableInterpreter() *ConfigurableInterpreter { + return &ConfigurableInterpreter{ + configManager: NewThirdPartyConfigManager(), + luaVM: luavm.New(false, 10), + } +} diff --git a/pkg/resourceinterpreter/interpreter.go b/pkg/resourceinterpreter/interpreter.go index f08442c27..212dc30fc 100644 --- a/pkg/resourceinterpreter/interpreter.go +++ b/pkg/resourceinterpreter/interpreter.go @@ -14,6 +14,7 @@ import ( "github.com/karmada-io/karmada/pkg/resourceinterpreter/customized/webhook" "github.com/karmada-io/karmada/pkg/resourceinterpreter/customized/webhook/request" "github.com/karmada-io/karmada/pkg/resourceinterpreter/default/native" + "github.com/karmada-io/karmada/pkg/resourceinterpreter/default/thirdparty" "github.com/karmada-io/karmada/pkg/util/fedinformer/genericmanager" ) @@ -59,9 +60,10 @@ func NewResourceInterpreter(informer genericmanager.SingleClusterInformerManager type customResourceInterpreterImpl struct { informer genericmanager.SingleClusterInformerManager - customizedInterpreter *webhook.CustomizedInterpreter - defaultInterpreter *native.DefaultInterpreter configurableInterpreter *declarative.ConfigurableInterpreter + customizedInterpreter *webhook.CustomizedInterpreter + thirdpartyInterpreter *thirdparty.ConfigurableInterpreter + defaultInterpreter *native.DefaultInterpreter } // Start starts running the component and will never stop running until the context is closed or an error occurs. @@ -74,6 +76,7 @@ func (i *customResourceInterpreterImpl) Start(ctx context.Context) (err error) { } i.configurableInterpreter = declarative.NewConfigurableInterpreter(i.informer) + i.thirdpartyInterpreter = thirdparty.NewConfigurableInterpreter() i.defaultInterpreter = native.NewDefaultInterpreter() i.informer.Start() @@ -85,6 +88,7 @@ func (i *customResourceInterpreterImpl) Start(ctx context.Context) (err error) { // HookEnabled tells if any hook exist for specific resource type and operation. func (i *customResourceInterpreterImpl) HookEnabled(objGVK schema.GroupVersionKind, operation configv1alpha1.InterpreterOperation) bool { return i.defaultInterpreter.HookEnabled(objGVK, operation) || + i.thirdpartyInterpreter.HookEnabled(objGVK, operation) || i.configurableInterpreter.HookEnabled(objGVK, operation) || i.customizedInterpreter.HookEnabled(objGVK, operation) } @@ -111,6 +115,13 @@ func (i *customResourceInterpreterImpl) GetReplicas(object *unstructured.Unstruc if hookEnabled { return } + replica, requires, hookEnabled, err = i.thirdpartyInterpreter.GetReplicas(object) + if err != nil { + return + } + if hookEnabled { + return + } replica, requires, err = i.defaultInterpreter.GetReplicas(object) return @@ -138,6 +149,13 @@ func (i *customResourceInterpreterImpl) ReviseReplica(object *unstructured.Unstr if hookEnabled { return obj, nil } + obj, hookEnabled, err = i.thirdpartyInterpreter.ReviseReplica(object, replica) + if err != nil { + return nil, err + } + if hookEnabled { + return obj, nil + } return i.defaultInterpreter.ReviseReplica(object, replica) } @@ -164,6 +182,13 @@ func (i *customResourceInterpreterImpl) Retain(desired *unstructured.Unstructure if hookEnabled { return obj, nil } + obj, hookEnabled, err = i.thirdpartyInterpreter.Retain(desired, observed) + if err != nil { + return nil, err + } + if hookEnabled { + return obj, nil + } return i.defaultInterpreter.Retain(desired, observed) } @@ -190,6 +215,13 @@ func (i *customResourceInterpreterImpl) AggregateStatus(object *unstructured.Uns if hookEnabled { return obj, nil } + obj, hookEnabled, err = i.thirdpartyInterpreter.AggregateStatus(object, aggregatedStatusItems) + if err != nil { + return nil, err + } + if hookEnabled { + return obj, nil + } return i.defaultInterpreter.AggregateStatus(object, aggregatedStatusItems) } @@ -214,6 +246,13 @@ func (i *customResourceInterpreterImpl) GetDependencies(object *unstructured.Uns if hookEnabled { return } + dependencies, hookEnabled, err = i.thirdpartyInterpreter.GetDependencies(object) + if err != nil { + return + } + if hookEnabled { + return + } dependencies, err = i.defaultInterpreter.GetDependencies(object) return @@ -238,7 +277,13 @@ func (i *customResourceInterpreterImpl) ReflectStatus(object *unstructured.Unstr if hookEnabled { return } - + status, hookEnabled, err = i.thirdpartyInterpreter.ReflectStatus(object) + if err != nil { + return + } + if hookEnabled { + return + } status, err = i.defaultInterpreter.ReflectStatus(object) return } @@ -263,6 +308,13 @@ func (i *customResourceInterpreterImpl) InterpretHealth(object *unstructured.Uns if hookEnabled { return } + healthy, hookEnabled, err = i.thirdpartyInterpreter.InterpretHealth(object) + if err != nil { + return + } + if hookEnabled { + return + } healthy, err = i.defaultInterpreter.InterpretHealth(object) return