From 805aca42d1840eadac4263450dfa157eed2ca34f Mon Sep 17 00:00:00 2001 From: calvin Date: Sun, 25 Jun 2023 23:40:31 +0800 Subject: [PATCH] support custom args and featureGates Signed-off-by: calvin --- .../pkg/controlplane/apiserver/apiserver.go | 19 +- operator/pkg/controlplane/controlplane.go | 31 +-- operator/pkg/controlplane/webhook/webhook.go | 9 +- operator/pkg/init.go | 4 + operator/pkg/tasks/init/apiserver.go | 14 +- operator/pkg/tasks/init/component.go | 9 +- operator/pkg/tasks/init/data.go | 1 + operator/pkg/util/patcher/pather.go | 187 +++++++++++++++++- 8 files changed, 242 insertions(+), 32 deletions(-) diff --git a/operator/pkg/controlplane/apiserver/apiserver.go b/operator/pkg/controlplane/apiserver/apiserver.go index b7b5b5887..d3f5a1ffb 100644 --- a/operator/pkg/controlplane/apiserver/apiserver.go +++ b/operator/pkg/controlplane/apiserver/apiserver.go @@ -17,8 +17,8 @@ import ( ) // EnsureKarmadaAPIServer creates karmada apiserver deployment and service resource -func EnsureKarmadaAPIServer(client clientset.Interface, cfg *operatorv1alpha1.KarmadaComponents, name, namespace string) error { - if err := installKarmadaAPIServer(client, cfg.KarmadaAPIServer, name, namespace); err != nil { +func EnsureKarmadaAPIServer(client clientset.Interface, cfg *operatorv1alpha1.KarmadaComponents, name, namespace string, featureGates map[string]bool) error { + if err := installKarmadaAPIServer(client, cfg.KarmadaAPIServer, name, namespace, featureGates); err != nil { return fmt.Errorf("failed to install karmada apiserver, err: %w", err) } @@ -26,14 +26,14 @@ func EnsureKarmadaAPIServer(client clientset.Interface, cfg *operatorv1alpha1.Ka } // EnsureKarmadaAggregatedAPIServer creates karmada aggregated apiserver deployment and service resource -func EnsureKarmadaAggregatedAPIServer(client clientset.Interface, cfg *operatorv1alpha1.KarmadaComponents, name, namespace string) error { - if err := installKarmadaAggregatedAPIServer(client, cfg.KarmadaAggregatedAPIServer, name, namespace); err != nil { +func EnsureKarmadaAggregatedAPIServer(client clientset.Interface, cfg *operatorv1alpha1.KarmadaComponents, name, namespace string, featureGates map[string]bool) error { + if err := installKarmadaAggregatedAPIServer(client, cfg.KarmadaAggregatedAPIServer, name, namespace, featureGates); err != nil { return err } return createKarmadaAggregatedAPIServerService(client, name, namespace) } -func installKarmadaAPIServer(client clientset.Interface, cfg *operatorv1alpha1.KarmadaAPIServer, name, namespace string) error { +func installKarmadaAPIServer(client clientset.Interface, cfg *operatorv1alpha1.KarmadaAPIServer, name, namespace string, featureGates map[string]bool) error { apiserverDeploymentBytes, err := util.ParseTemplate(KarmadaApiserverDeployment, struct { DeploymentName, Namespace, Image, EtcdClientService string ServiceSubnet, KarmadaCertsSecret, EtcdCertsSecret string @@ -58,8 +58,8 @@ func installKarmadaAPIServer(client clientset.Interface, cfg *operatorv1alpha1.K if err := kuberuntime.DecodeInto(clientsetscheme.Codecs.UniversalDecoder(), apiserverDeploymentBytes, apiserverDeployment); err != nil { return fmt.Errorf("error when decoding karmadaApiserver deployment: %w", err) } - - patcher.NewPatcher().WithAnnotations(cfg.Annotations).WithLabels(cfg.Labels).ForDeployment(apiserverDeployment) + patcher.NewPatcher().WithAnnotations(cfg.Annotations).WithLabels(cfg.Labels). + WithExtraArgs(cfg.ExtraArgs).ForDeployment(apiserverDeployment) if err := apiclient.CreateOrUpdateDeployment(client, apiserverDeployment); err != nil { return fmt.Errorf("error when creating deployment for %s, err: %w", apiserverDeployment.Name, err) @@ -90,7 +90,7 @@ func createKarmadaAPIServerService(client clientset.Interface, cfg *operatorv1al return nil } -func installKarmadaAggregatedAPIServer(client clientset.Interface, cfg *operatorv1alpha1.KarmadaAggregatedAPIServer, name, namespace string) error { +func installKarmadaAggregatedAPIServer(client clientset.Interface, cfg *operatorv1alpha1.KarmadaAggregatedAPIServer, name, namespace string, featureGates map[string]bool) error { aggregatedAPIServerDeploymentBytes, err := util.ParseTemplate(KarmadaAggregatedAPIServerDeployment, struct { DeploymentName, Namespace, Image, EtcdClientService string KubeconfigSecret, KarmadaCertsSecret, EtcdCertsSecret string @@ -116,7 +116,8 @@ func installKarmadaAggregatedAPIServer(client clientset.Interface, cfg *operator return fmt.Errorf("err when decoding karmadaApiserver deployment: %w", err) } - patcher.NewPatcher().WithAnnotations(cfg.Annotations).WithLabels(cfg.Labels).ForDeployment(aggregatedAPIServerDeployment) + patcher.NewPatcher().WithAnnotations(cfg.Annotations).WithLabels(cfg.Labels). + WithExtraArgs(cfg.ExtraArgs).WithFeatureGates(featureGates).ForDeployment(aggregatedAPIServerDeployment) if err := apiclient.CreateOrUpdateDeployment(client, aggregatedAPIServerDeployment); err != nil { return fmt.Errorf("error when creating deployment for %s, err: %w", aggregatedAPIServerDeployment.Name, err) diff --git a/operator/pkg/controlplane/controlplane.go b/operator/pkg/controlplane/controlplane.go index 020a06525..27b0b632f 100644 --- a/operator/pkg/controlplane/controlplane.go +++ b/operator/pkg/controlplane/controlplane.go @@ -17,8 +17,8 @@ import ( ) // EnsureControlPlaneComponent creates karmada controllerManager, kubeControllerManager, scheduler, webhook component -func EnsureControlPlaneComponent(component, name, namespace string, client clientset.Interface, cfg *operatorv1alpha1.KarmadaComponents) error { - deployments, err := getComponentManifests(name, namespace, cfg) +func EnsureControlPlaneComponent(component, name, namespace string, featureGates map[string]bool, client clientset.Interface, cfg *operatorv1alpha1.KarmadaComponents) error { + deployments, err := getComponentManifests(name, namespace, featureGates, cfg) if err != nil { return err } @@ -35,16 +35,16 @@ func EnsureControlPlaneComponent(component, name, namespace string, client clien return nil } -func getComponentManifests(name, namespace string, cfg *operatorv1alpha1.KarmadaComponents) (map[string]*appsv1.Deployment, error) { +func getComponentManifests(name, namespace string, featureGates map[string]bool, cfg *operatorv1alpha1.KarmadaComponents) (map[string]*appsv1.Deployment, error) { kubeControllerManager, err := getKubeControllerManagerManifest(name, namespace, cfg.KubeControllerManager) if err != nil { return nil, err } - karmadaControllerManager, err := getKarmadaControllerManagerManifest(name, namespace, cfg.KarmadaControllerManager) + karmadaControllerManager, err := getKarmadaControllerManagerManifest(name, namespace, featureGates, cfg.KarmadaControllerManager) if err != nil { return nil, err } - karmadaScheduler, err := getKarmadaSchedulerManifest(name, namespace, cfg.KarmadaScheduler) + karmadaScheduler, err := getKarmadaSchedulerManifest(name, namespace, featureGates, cfg.KarmadaScheduler) if err != nil { return nil, err } @@ -56,7 +56,7 @@ func getComponentManifests(name, namespace string, cfg *operatorv1alpha1.Karmada } if cfg.KarmadaDescheduler != nil { - descheduler, err := getKarmadaDeschedulerManifest(name, namespace, cfg.KarmadaDescheduler) + descheduler, err := getKarmadaDeschedulerManifest(name, namespace, featureGates, cfg.KarmadaDescheduler) if err != nil { return nil, err } @@ -88,11 +88,12 @@ func getKubeControllerManagerManifest(name, namespace string, cfg *operatorv1alp return nil, fmt.Errorf("err when decoding kube-controller-manager deployment: %w", err) } - patcher.NewPatcher().WithAnnotations(cfg.Annotations).WithLabels(cfg.Labels).ForDeployment(kcm) + patcher.NewPatcher().WithAnnotations(cfg.Annotations). + WithLabels(cfg.Labels).WithExtraArgs(cfg.ExtraArgs).ForDeployment(kcm) return kcm, nil } -func getKarmadaControllerManagerManifest(name, namespace string, cfg *operatorv1alpha1.KarmadaControllerManager) (*appsv1.Deployment, error) { +func getKarmadaControllerManagerManifest(name, namespace string, featureGates map[string]bool, cfg *operatorv1alpha1.KarmadaControllerManager) (*appsv1.Deployment, error) { karmadaControllerManagerBytes, err := util.ParseTemplate(KamradaControllerManagerDeployment, struct { Replicas *int32 DeploymentName, Namespace string @@ -113,11 +114,12 @@ func getKarmadaControllerManagerManifest(name, namespace string, cfg *operatorv1 return nil, fmt.Errorf("err when decoding karmada-controller-manager deployment: %w", err) } - patcher.NewPatcher().WithAnnotations(cfg.Annotations).WithLabels(cfg.Labels).ForDeployment(kcm) + patcher.NewPatcher().WithAnnotations(cfg.Annotations).WithLabels(cfg.Labels). + WithExtraArgs(cfg.ExtraArgs).WithFeatureGates(featureGates).ForDeployment(kcm) return kcm, nil } -func getKarmadaSchedulerManifest(name, namespace string, cfg *operatorv1alpha1.KarmadaScheduler) (*appsv1.Deployment, error) { +func getKarmadaSchedulerManifest(name, namespace string, featureGates map[string]bool, cfg *operatorv1alpha1.KarmadaScheduler) (*appsv1.Deployment, error) { karmadaSchedulerBytes, err := util.ParseTemplate(KarmadaSchedulerDeployment, struct { Replicas *int32 DeploymentName, Namespace string @@ -138,11 +140,12 @@ func getKarmadaSchedulerManifest(name, namespace string, cfg *operatorv1alpha1.K return nil, fmt.Errorf("err when decoding karmada-scheduler deployment: %w", err) } - patcher.NewPatcher().WithAnnotations(cfg.Annotations).WithLabels(cfg.Labels).ForDeployment(scheduler) + patcher.NewPatcher().WithAnnotations(cfg.Annotations).WithLabels(cfg.Labels). + WithExtraArgs(cfg.ExtraArgs).WithFeatureGates(featureGates).ForDeployment(scheduler) return scheduler, nil } -func getKarmadaDeschedulerManifest(name, namespace string, cfg *operatorv1alpha1.KarmadaDescheduler) (*appsv1.Deployment, error) { +func getKarmadaDeschedulerManifest(name, namespace string, featureGates map[string]bool, cfg *operatorv1alpha1.KarmadaDescheduler) (*appsv1.Deployment, error) { karmadaDeschedulerBytes, err := util.ParseTemplate(KarmadaDeschedulerDeployment, struct { Replicas *int32 DeploymentName, Namespace string @@ -163,6 +166,8 @@ func getKarmadaDeschedulerManifest(name, namespace string, cfg *operatorv1alpha1 return nil, fmt.Errorf("err when decoding karmada-descheduler deployment: %w", err) } - patcher.NewPatcher().WithAnnotations(cfg.Annotations).WithLabels(cfg.Labels).ForDeployment(descheduler) + patcher.NewPatcher().WithAnnotations(cfg.Annotations).WithLabels(cfg.Labels). + WithExtraArgs(cfg.ExtraArgs).WithFeatureGates(featureGates).ForDeployment(descheduler) + return descheduler, nil } diff --git a/operator/pkg/controlplane/webhook/webhook.go b/operator/pkg/controlplane/webhook/webhook.go index 1bc5eb254..20c8ba208 100644 --- a/operator/pkg/controlplane/webhook/webhook.go +++ b/operator/pkg/controlplane/webhook/webhook.go @@ -16,15 +16,15 @@ import ( ) // EnsureKarmadaWebhook creates karmada webhook deployment and service resource. -func EnsureKarmadaWebhook(client clientset.Interface, cfg *operatorv1alpha1.KarmadaWebhook, name, namespace string) error { - if err := installKarmadaWebhook(client, cfg, name, namespace); err != nil { +func EnsureKarmadaWebhook(client clientset.Interface, cfg *operatorv1alpha1.KarmadaWebhook, name, namespace string, featureGates map[string]bool) error { + if err := installKarmadaWebhook(client, cfg, name, namespace, featureGates); err != nil { return err } return createKarmadaWebhookService(client, name, namespace) } -func installKarmadaWebhook(client clientset.Interface, cfg *operatorv1alpha1.KarmadaWebhook, name, namespace string) error { +func installKarmadaWebhook(client clientset.Interface, cfg *operatorv1alpha1.KarmadaWebhook, name, namespace string, featureGates map[string]bool) error { webhookDeploymentSetBytes, err := util.ParseTemplate(KarmadaWebhookDeployment, struct { DeploymentName, Namespace, Image string KubeconfigSecret, WebhookCertsSecret string @@ -46,7 +46,8 @@ func installKarmadaWebhook(client clientset.Interface, cfg *operatorv1alpha1.Kar return fmt.Errorf("err when decoding KarmadaWebhook Deployment: %w", err) } - patcher.NewPatcher().WithAnnotations(cfg.Annotations).WithLabels(cfg.Labels).ForDeployment(webhookDeployment) + patcher.NewPatcher().WithAnnotations(cfg.Annotations).WithLabels(cfg.Labels). + WithExtraArgs(cfg.ExtraArgs).ForDeployment(webhookDeployment) if err := apiclient.CreateOrUpdateDeployment(client, webhookDeployment); err != nil { return fmt.Errorf("error when creating deployment for %s, err: %w", webhookDeployment.Name, err) diff --git a/operator/pkg/init.go b/operator/pkg/init.go index 142a4ebd8..e0858eaaf 100644 --- a/operator/pkg/init.go +++ b/operator/pkg/init.go @@ -207,6 +207,10 @@ func (data *initData) ControlplaneAddress() string { return data.controlplaneAddress } +func (data *initData) FeatureGates() map[string]bool { + return data.featureGates +} + // NewJobInitOptions calls all of InitOpt func to initialize a InitOptions. // if there is not InitOpt functions, it will return a default InitOptions. func NewJobInitOptions(opts ...InitOpt) *InitOptions { diff --git a/operator/pkg/tasks/init/apiserver.go b/operator/pkg/tasks/init/apiserver.go index 8a712e7d7..69dbebfa1 100644 --- a/operator/pkg/tasks/init/apiserver.go +++ b/operator/pkg/tasks/init/apiserver.go @@ -84,7 +84,12 @@ func runKarmadaAPIServer(r workflow.RunData) error { return nil } - err := apiserver.EnsureKarmadaAPIServer(data.RemoteClient(), cfg, data.GetName(), data.GetNamespace()) + err := apiserver.EnsureKarmadaAPIServer( + data.RemoteClient(), + cfg, + data.GetName(), + data.GetNamespace(), + data.FeatureGates()) if err != nil { return fmt.Errorf("failed to install karmada apiserver component, err: %w", err) } @@ -122,7 +127,12 @@ func runKarmadaAggregatedAPIServer(r workflow.RunData) error { return nil } - err := apiserver.EnsureKarmadaAggregatedAPIServer(data.RemoteClient(), cfg, data.GetName(), data.GetNamespace()) + err := apiserver.EnsureKarmadaAggregatedAPIServer( + data.RemoteClient(), + cfg, + data.GetName(), + data.GetNamespace(), + data.FeatureGates()) if err != nil { return fmt.Errorf("failed to install karmada aggregated apiserver, err: %w", err) } diff --git a/operator/pkg/tasks/init/component.go b/operator/pkg/tasks/init/component.go index 42715436e..7d01080f7 100644 --- a/operator/pkg/tasks/init/component.go +++ b/operator/pkg/tasks/init/component.go @@ -59,6 +59,7 @@ func runComponentSubTask(component string) func(r workflow.RunData) error { component, data.GetName(), data.GetNamespace(), + data.FeatureGates(), data.RemoteClient(), data.Components(), ) @@ -82,7 +83,13 @@ func runKarmadaWebhook(r workflow.RunData) error { return errors.New("skip install karmada webhook") } - err := webhook.EnsureKarmadaWebhook(data.RemoteClient(), cfg.KarmadaWebhook, data.GetName(), data.GetNamespace()) + err := webhook.EnsureKarmadaWebhook( + data.RemoteClient(), + cfg.KarmadaWebhook, + data.GetName(), + data.GetNamespace(), + data.FeatureGates(), + ) if err != nil { return fmt.Errorf("failed to apply karmada webhook, err: %w", err) } diff --git a/operator/pkg/tasks/init/data.go b/operator/pkg/tasks/init/data.go index 5c5e9699e..706af7236 100644 --- a/operator/pkg/tasks/init/data.go +++ b/operator/pkg/tasks/init/data.go @@ -22,4 +22,5 @@ type InitData interface { CrdsRemoteURL() string KarmadaVersion() string Components() *operatorv1alpha1.KarmadaComponents + FeatureGates() map[string]bool } diff --git a/operator/pkg/util/patcher/pather.go b/operator/pkg/util/patcher/pather.go index 5ed1d5eef..b36fbfe6b 100644 --- a/operator/pkg/util/patcher/pather.go +++ b/operator/pkg/util/patcher/pather.go @@ -1,10 +1,17 @@ package patcher import ( + "errors" + "fmt" + "sort" + "strconv" + "strings" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" + "k8s.io/klog/v2" operatorv1alpha1 "github.com/karmada-io/karmada/operator/pkg/apis/operator/v1alpha1" "github.com/karmada-io/karmada/operator/pkg/constants" @@ -12,9 +19,11 @@ import ( // Patcher defines multiple variables that need to be patched. type Patcher struct { - labels map[string]string - annotations map[string]string - volume *operatorv1alpha1.VolumeData + labels map[string]string + annotations map[string]string + extraArgs map[string]string + featureGates map[string]bool + volume *operatorv1alpha1.VolumeData } // NewPatcher returns a patcher. @@ -34,6 +43,18 @@ func (p *Patcher) WithAnnotations(annotations labels.Set) *Patcher { return p } +// WithExtraArgs sets extraArgs to the patcher. +func (p *Patcher) WithExtraArgs(extraArgs map[string]string) *Patcher { + p.extraArgs = extraArgs + return p +} + +// WithFeatureGates sets featureGates to the patcher. +func (p *Patcher) WithFeatureGates(featureGates map[string]bool) *Patcher { + p.featureGates = featureGates + return p +} + // WithVolumeData sets VolumeData to the patcher. func (p *Patcher) WithVolumeData(volume *operatorv1alpha1.VolumeData) *Patcher { p.volume = volume @@ -47,6 +68,33 @@ func (p *Patcher) ForDeployment(deployment *appsv1.Deployment) { deployment.Annotations = labels.Merge(deployment.Annotations, p.annotations) deployment.Spec.Template.Annotations = labels.Merge(deployment.Spec.Template.Annotations, p.annotations) + + if len(p.extraArgs) != 0 || len(p.featureGates) != 0 { + // It's considered the first container is the karmada component by default. + baseArguments := deployment.Spec.Template.Spec.Containers[0].Command + argsMap := parseArgumentListToMap(baseArguments) + + overrideArgs := map[string]string{} + + // merge featureGates and build to an argurment. + if len(p.featureGates) != 0 { + baseFeatureGates := map[string]bool{} + + if argument, ok := argsMap["feature-gates"]; ok { + baseFeatureGates = parseFeatrueGatesArgumentToMap(argument) + } + overrideArgs["feature-gates"] = buildFeatureGatesArgumentFromMap(baseFeatureGates, p.featureGates) + } + + for key, val := range p.extraArgs { + overrideArgs[key] = val + } + + // the first argument is most often the binary name + command := []string{baseArguments[0]} + command = append(command, buildArgumentListFromMap(argsMap, overrideArgs)...) + deployment.Spec.Template.Spec.Containers[0].Command = command + } } // ForStatefulSet patches the statefulset manifest. @@ -60,6 +108,139 @@ func (p *Patcher) ForStatefulSet(sts *appsv1.StatefulSet) { if p.volume != nil { patchVolumeForStatefulSet(sts, p.volume) } + + if len(p.extraArgs) != 0 { + // It's considered the first container is the karmada component by default. + baseArguments := sts.Spec.Template.Spec.Containers[0].Command + argsMap := parseArgumentListToMap(baseArguments) + sts.Spec.Template.Spec.Containers[0].Command = buildArgumentListFromMap(argsMap, p.extraArgs) + } +} + +func buildArgumentListFromMap(baseArguments, overrideArguments map[string]string) []string { + var command []string + var keys []string + + argsMap := make(map[string]string) + + for k, v := range baseArguments { + argsMap[k] = v + } + for k, v := range overrideArguments { + argsMap[k] = v + } + + for k := range argsMap { + keys = append(keys, k) + } + + sort.Strings(keys) + for _, k := range keys { + command = append(command, fmt.Sprintf("--%s=%s", k, argsMap[k])) + } + + return command +} + +func parseFeatrueGatesArgumentToMap(featureGates string) map[string]bool { + featureGateSlice := strings.Split(featureGates, ",") + + featureGatesMap := map[string]bool{} + for _, featureGate := range featureGateSlice { + key, val, err := parseFeatrueGate(featureGate) + if err != nil { + continue + } + featureGatesMap[key] = val + } + + return featureGatesMap +} + +func buildFeatureGatesArgumentFromMap(baseFeatureGates, overrideFeatureGates map[string]bool) string { + var featureGates []string + var keys []string + + featureGateMap := make(map[string]bool) + + for k, v := range baseFeatureGates { + featureGateMap[k] = v + } + for k, v := range overrideFeatureGates { + featureGateMap[k] = v + } + + for k := range featureGateMap { + keys = append(keys, k) + } + + sort.Strings(keys) + for _, k := range keys { + featureGates = append(featureGates, fmt.Sprintf("%s=%s", k, strconv.FormatBool(featureGateMap[k]))) + } + + return strings.Join(featureGates, ",") +} + +func parseArgumentListToMap(arguments []string) map[string]string { + resultingMap := map[string]string{} + for i, arg := range arguments { + key, val, err := parseArgument(arg) + // Ignore if the first argument doesn't satisfy the criteria, it's most often the binary name + // Warn in all other cases, but don't error out. This can happen only if the user has edited the argument list by hand, so they might know what they are doing + if err != nil { + if i != 0 { + klog.Warningf("WARNING: The component argument %q could not be parsed correctly. The argument must be of the form %q. Skipping...\n", arg, "--") + } + continue + } + + resultingMap[key] = val + } + return resultingMap +} + +func parseArgument(arg string) (string, string, error) { + if !strings.HasPrefix(arg, "--") { + return "", "", errors.New("the argument should start with '--'") + } + if !strings.Contains(arg, "=") { + return "", "", errors.New("the argument should have a '=' between the flag and the value") + } + + arg = strings.TrimPrefix(arg, "--") + keyvalSlice := strings.SplitN(arg, "=", 2) + + if len(keyvalSlice) != 2 { + return "", "", errors.New("the argument must have both a key and a value") + } + if len(keyvalSlice[0]) == 0 { + return "", "", errors.New("the argument must have a key") + } + + return keyvalSlice[0], keyvalSlice[1], nil +} + +func parseFeatrueGate(featureGate string) (string, bool, error) { + if !strings.Contains(featureGate, "=") { + return "", false, errors.New("the featureGate should have a '=' between the flag and the value") + } + + keyvalSlice := strings.SplitN(featureGate, "=", 2) + + if len(keyvalSlice) != 2 { + return "", false, errors.New("the featureGate must have both a key and a value") + } + if len(keyvalSlice[0]) == 0 { + return "", false, errors.New("the featureGate must have a key") + } + + val, err := strconv.ParseBool(keyvalSlice[1]) + if err != nil { + return "", false, errors.New("the featureGate value must have a value of type bool") + } + + return keyvalSlice[0], val, nil } func patchVolumeForStatefulSet(sts *appsv1.StatefulSet, volume *operatorv1alpha1.VolumeData) {