karmada-io
/
karmada
mirror of https://github.com/karmada-io/karmada.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

support custom args and featureGates 

Signed-off-by: calvin <wen.chen@daocloud.io>
This commit is contained in:
calvin 2023-06-25 23:40:31 +08:00
parent 33f3016853
commit 805aca42d1
8 changed files with 242 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
operator/pkg/controlplane/apiserver/apiserver.go
View File

@ -17,8 +17,8 @@ import (
)
// EnsureKarmadaAPIServer creates karmada apiserver deployment and service resource
func EnsureKarmadaAPIServer(client clientset.Interface, cfg *operatorv1alpha1.KarmadaComponents, name, namespace string) error {
if err := installKarmadaAPIServer(client, cfg.KarmadaAPIServer, name, namespace); err != nil {
func EnsureKarmadaAPIServer(client clientset.Interface, cfg *operatorv1alpha1.KarmadaComponents, name, namespace string, featureGates map[string]bool) error {
if err := installKarmadaAPIServer(client, cfg.KarmadaAPIServer, name, namespace, featureGates); err != nil {
return fmt.Errorf("failed to install karmada apiserver, err: %w", err)
}
@ -26,14 +26,14 @@ func EnsureKarmadaAPIServer(client clientset.Interface, cfg *operatorv1alpha1.Ka
}
// EnsureKarmadaAggregatedAPIServer creates karmada aggregated apiserver deployment and service resource
func EnsureKarmadaAggregatedAPIServer(client clientset.Interface, cfg *operatorv1alpha1.KarmadaComponents, name, namespace string) error {
if err := installKarmadaAggregatedAPIServer(client, cfg.KarmadaAggregatedAPIServer, name, namespace); err != nil {
func EnsureKarmadaAggregatedAPIServer(client clientset.Interface, cfg *operatorv1alpha1.KarmadaComponents, name, namespace string, featureGates map[string]bool) error {
if err := installKarmadaAggregatedAPIServer(client, cfg.KarmadaAggregatedAPIServer, name, namespace, featureGates); err != nil {
return err
}
return createKarmadaAggregatedAPIServerService(client, name, namespace)
}
func installKarmadaAPIServer(client clientset.Interface, cfg *operatorv1alpha1.KarmadaAPIServer, name, namespace string) error {
func installKarmadaAPIServer(client clientset.Interface, cfg *operatorv1alpha1.KarmadaAPIServer, name, namespace string, featureGates map[string]bool) error {
apiserverDeploymentBytes, err := util.ParseTemplate(KarmadaApiserverDeployment, struct {
DeploymentName, Namespace, Image, EtcdClientService string
ServiceSubnet, KarmadaCertsSecret, EtcdCertsSecret string
@ -58,8 +58,8 @@ func installKarmadaAPIServer(client clientset.Interface, cfg *operatorv1alpha1.K
if err := kuberuntime.DecodeInto(clientsetscheme.Codecs.UniversalDecoder(), apiserverDeploymentBytes, apiserverDeployment); err != nil {
return fmt.Errorf("error when decoding karmadaApiserver deployment: %w", err)
}
patcher.NewPatcher().WithAnnotations(cfg.Annotations).WithLabels(cfg.Labels).ForDeployment(apiserverDeployment)
patcher.NewPatcher().WithAnnotations(cfg.Annotations).WithLabels(cfg.Labels).
WithExtraArgs(cfg.ExtraArgs).ForDeployment(apiserverDeployment)
if err := apiclient.CreateOrUpdateDeployment(client, apiserverDeployment); err != nil {
return fmt.Errorf("error when creating deployment for %s, err: %w", apiserverDeployment.Name, err)
@ -90,7 +90,7 @@ func createKarmadaAPIServerService(client clientset.Interface, cfg *operatorv1al
return nil
}
func installKarmadaAggregatedAPIServer(client clientset.Interface, cfg *operatorv1alpha1.KarmadaAggregatedAPIServer, name, namespace string) error {
func installKarmadaAggregatedAPIServer(client clientset.Interface, cfg *operatorv1alpha1.KarmadaAggregatedAPIServer, name, namespace string, featureGates map[string]bool) error {
aggregatedAPIServerDeploymentBytes, err := util.ParseTemplate(KarmadaAggregatedAPIServerDeployment, struct {
DeploymentName, Namespace, Image, EtcdClientService string
KubeconfigSecret, KarmadaCertsSecret, EtcdCertsSecret string
@ -116,7 +116,8 @@ func installKarmadaAggregatedAPIServer(client clientset.Interface, cfg *operator
return fmt.Errorf("err when decoding karmadaApiserver deployment: %w", err)
}
patcher.NewPatcher().WithAnnotations(cfg.Annotations).WithLabels(cfg.Labels).ForDeployment(aggregatedAPIServerDeployment)
patcher.NewPatcher().WithAnnotations(cfg.Annotations).WithLabels(cfg.Labels).
WithExtraArgs(cfg.ExtraArgs).WithFeatureGates(featureGates).ForDeployment(aggregatedAPIServerDeployment)
if err := apiclient.CreateOrUpdateDeployment(client, aggregatedAPIServerDeployment); err != nil {
return fmt.Errorf("error when creating deployment for %s, err: %w", aggregatedAPIServerDeployment.Name, err)

31
operator/pkg/controlplane/controlplane.go
View File

@ -17,8 +17,8 @@ import (
)
// EnsureControlPlaneComponent creates karmada controllerManager, kubeControllerManager, scheduler, webhook component
func EnsureControlPlaneComponent(component, name, namespace string, client clientset.Interface, cfg *operatorv1alpha1.KarmadaComponents) error {
deployments, err := getComponentManifests(name, namespace, cfg)
func EnsureControlPlaneComponent(component, name, namespace string, featureGates map[string]bool, client clientset.Interface, cfg *operatorv1alpha1.KarmadaComponents) error {
deployments, err := getComponentManifests(name, namespace, featureGates, cfg)
if err != nil {
return err
}
@ -35,16 +35,16 @@ func EnsureControlPlaneComponent(component, name, namespace string, client clien
return nil
}
func getComponentManifests(name, namespace string, cfg *operatorv1alpha1.KarmadaComponents) (map[string]*appsv1.Deployment, error) {
func getComponentManifests(name, namespace string, featureGates map[string]bool, cfg *operatorv1alpha1.KarmadaComponents) (map[string]*appsv1.Deployment, error) {
kubeControllerManager, err := getKubeControllerManagerManifest(name, namespace, cfg.KubeControllerManager)
if err != nil {
return nil, err
}
karmadaControllerManager, err := getKarmadaControllerManagerManifest(name, namespace, cfg.KarmadaControllerManager)
karmadaControllerManager, err := getKarmadaControllerManagerManifest(name, namespace, featureGates, cfg.KarmadaControllerManager)
if err != nil {
return nil, err
}
karmadaScheduler, err := getKarmadaSchedulerManifest(name, namespace, cfg.KarmadaScheduler)
karmadaScheduler, err := getKarmadaSchedulerManifest(name, namespace, featureGates, cfg.KarmadaScheduler)
if err != nil {
return nil, err
}
@ -56,7 +56,7 @@ func getComponentManifests(name, namespace string, cfg *operatorv1alpha1.Karmada
}
if cfg.KarmadaDescheduler != nil {
descheduler, err := getKarmadaDeschedulerManifest(name, namespace, cfg.KarmadaDescheduler)
descheduler, err := getKarmadaDeschedulerManifest(name, namespace, featureGates, cfg.KarmadaDescheduler)
if err != nil {
return nil, err
}
@ -88,11 +88,12 @@ func getKubeControllerManagerManifest(name, namespace string, cfg *operatorv1alp
return nil, fmt.Errorf("err when decoding kube-controller-manager deployment: %w", err)
}
patcher.NewPatcher().WithAnnotations(cfg.Annotations).WithLabels(cfg.Labels).ForDeployment(kcm)
patcher.NewPatcher().WithAnnotations(cfg.Annotations).
WithLabels(cfg.Labels).WithExtraArgs(cfg.ExtraArgs).ForDeployment(kcm)
return kcm, nil
}
func getKarmadaControllerManagerManifest(name, namespace string, cfg *operatorv1alpha1.KarmadaControllerManager) (*appsv1.Deployment, error) {
func getKarmadaControllerManagerManifest(name, namespace string, featureGates map[string]bool, cfg *operatorv1alpha1.KarmadaControllerManager) (*appsv1.Deployment, error) {
karmadaControllerManagerBytes, err := util.ParseTemplate(KamradaControllerManagerDeployment, struct {
Replicas *int32
DeploymentName, Namespace string
@ -113,11 +114,12 @@ func getKarmadaControllerManagerManifest(name, namespace string, cfg *operatorv1
return nil, fmt.Errorf("err when decoding karmada-controller-manager deployment: %w", err)
}
patcher.NewPatcher().WithAnnotations(cfg.Annotations).WithLabels(cfg.Labels).ForDeployment(kcm)
patcher.NewPatcher().WithAnnotations(cfg.Annotations).WithLabels(cfg.Labels).
WithExtraArgs(cfg.ExtraArgs).WithFeatureGates(featureGates).ForDeployment(kcm)
return kcm, nil
}
func getKarmadaSchedulerManifest(name, namespace string, cfg *operatorv1alpha1.KarmadaScheduler) (*appsv1.Deployment, error) {
func getKarmadaSchedulerManifest(name, namespace string, featureGates map[string]bool, cfg *operatorv1alpha1.KarmadaScheduler) (*appsv1.Deployment, error) {
karmadaSchedulerBytes, err := util.ParseTemplate(KarmadaSchedulerDeployment, struct {
Replicas *int32
DeploymentName, Namespace string
@ -138,11 +140,12 @@ func getKarmadaSchedulerManifest(name, namespace string, cfg *operatorv1alpha1.K
return nil, fmt.Errorf("err when decoding karmada-scheduler deployment: %w", err)
}
patcher.NewPatcher().WithAnnotations(cfg.Annotations).WithLabels(cfg.Labels).ForDeployment(scheduler)
patcher.NewPatcher().WithAnnotations(cfg.Annotations).WithLabels(cfg.Labels).
WithExtraArgs(cfg.ExtraArgs).WithFeatureGates(featureGates).ForDeployment(scheduler)
return scheduler, nil
}
func getKarmadaDeschedulerManifest(name, namespace string, cfg *operatorv1alpha1.KarmadaDescheduler) (*appsv1.Deployment, error) {
func getKarmadaDeschedulerManifest(name, namespace string, featureGates map[string]bool, cfg *operatorv1alpha1.KarmadaDescheduler) (*appsv1.Deployment, error) {
karmadaDeschedulerBytes, err := util.ParseTemplate(KarmadaDeschedulerDeployment, struct {
Replicas *int32
DeploymentName, Namespace string
@ -163,6 +166,8 @@ func getKarmadaDeschedulerManifest(name, namespace string, cfg *operatorv1alpha1
return nil, fmt.Errorf("err when decoding karmada-descheduler deployment: %w", err)
}
patcher.NewPatcher().WithAnnotations(cfg.Annotations).WithLabels(cfg.Labels).ForDeployment(descheduler)
patcher.NewPatcher().WithAnnotations(cfg.Annotations).WithLabels(cfg.Labels).
WithExtraArgs(cfg.ExtraArgs).WithFeatureGates(featureGates).ForDeployment(descheduler)
return descheduler, nil
}

9
operator/pkg/controlplane/webhook/webhook.go
View File

@ -16,15 +16,15 @@ import (
)
// EnsureKarmadaWebhook creates karmada webhook deployment and service resource.
func EnsureKarmadaWebhook(client clientset.Interface, cfg *operatorv1alpha1.KarmadaWebhook, name, namespace string) error {
if err := installKarmadaWebhook(client, cfg, name, namespace); err != nil {
func EnsureKarmadaWebhook(client clientset.Interface, cfg *operatorv1alpha1.KarmadaWebhook, name, namespace string, featureGates map[string]bool) error {
if err := installKarmadaWebhook(client, cfg, name, namespace, featureGates); err != nil {
return err
}
return createKarmadaWebhookService(client, name, namespace)
}
func installKarmadaWebhook(client clientset.Interface, cfg *operatorv1alpha1.KarmadaWebhook, name, namespace string) error {
func installKarmadaWebhook(client clientset.Interface, cfg *operatorv1alpha1.KarmadaWebhook, name, namespace string, featureGates map[string]bool) error {
webhookDeploymentSetBytes, err := util.ParseTemplate(KarmadaWebhookDeployment, struct {
DeploymentName, Namespace, Image string
KubeconfigSecret, WebhookCertsSecret string
@ -46,7 +46,8 @@ func installKarmadaWebhook(client clientset.Interface, cfg *operatorv1alpha1.Kar
return fmt.Errorf("err when decoding KarmadaWebhook Deployment: %w", err)
}
patcher.NewPatcher().WithAnnotations(cfg.Annotations).WithLabels(cfg.Labels).ForDeployment(webhookDeployment)
patcher.NewPatcher().WithAnnotations(cfg.Annotations).WithLabels(cfg.Labels).
WithExtraArgs(cfg.ExtraArgs).ForDeployment(webhookDeployment)
if err := apiclient.CreateOrUpdateDeployment(client, webhookDeployment); err != nil {
return fmt.Errorf("error when creating deployment for %s, err: %w", webhookDeployment.Name, err)

4
operator/pkg/init.go
View File

@ -207,6 +207,10 @@ func (data *initData) ControlplaneAddress() string {
return data.controlplaneAddress
}
func (data *initData) FeatureGates() map[string]bool {
return data.featureGates
}
// NewJobInitOptions calls all of InitOpt func to initialize a InitOptions.
// if there is not InitOpt functions, it will return a default InitOptions.
func NewJobInitOptions(opts ...InitOpt) *InitOptions {

14
operator/pkg/tasks/init/apiserver.go
View File

@ -84,7 +84,12 @@ func runKarmadaAPIServer(r workflow.RunData) error {
return nil
}
err := apiserver.EnsureKarmadaAPIServer(data.RemoteClient(), cfg, data.GetName(), data.GetNamespace())
err := apiserver.EnsureKarmadaAPIServer(
data.RemoteClient(),
cfg,
data.GetName(),
data.GetNamespace(),
data.FeatureGates())
if err != nil {
return fmt.Errorf("failed to install karmada apiserver component, err: %w", err)
}
@ -122,7 +127,12 @@ func runKarmadaAggregatedAPIServer(r workflow.RunData) error {
return nil
}
err := apiserver.EnsureKarmadaAggregatedAPIServer(data.RemoteClient(), cfg, data.GetName(), data.GetNamespace())
err := apiserver.EnsureKarmadaAggregatedAPIServer(
data.RemoteClient(),
cfg,
data.GetName(),
data.GetNamespace(),
data.FeatureGates())
if err != nil {
return fmt.Errorf("failed to install karmada aggregated apiserver, err: %w", err)
}

9
operator/pkg/tasks/init/component.go
View File

@ -59,6 +59,7 @@ func runComponentSubTask(component string) func(r workflow.RunData) error {
component,
data.GetName(),
data.GetNamespace(),
data.FeatureGates(),
data.RemoteClient(),
data.Components(),
)
@ -82,7 +83,13 @@ func runKarmadaWebhook(r workflow.RunData) error {
return errors.New("skip install karmada webhook")
}
err := webhook.EnsureKarmadaWebhook(data.RemoteClient(), cfg.KarmadaWebhook, data.GetName(), data.GetNamespace())
err := webhook.EnsureKarmadaWebhook(
data.RemoteClient(),
cfg.KarmadaWebhook,
data.GetName(),
data.GetNamespace(),
data.FeatureGates(),
)
if err != nil {
return fmt.Errorf("failed to apply karmada webhook, err: %w", err)
}

1
operator/pkg/tasks/init/data.go
View File

@ -22,4 +22,5 @@ type InitData interface {
CrdsRemoteURL() string
KarmadaVersion() string
Components() *operatorv1alpha1.KarmadaComponents
FeatureGates() map[string]bool
}

187
operator/pkg/util/patcher/pather.go
View File

@ -1,10 +1,17 @@
package patcher
import (
"errors"
"fmt"
"sort"
"strconv"
"strings"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/klog/v2"
operatorv1alpha1 "github.com/karmada-io/karmada/operator/pkg/apis/operator/v1alpha1"
"github.com/karmada-io/karmada/operator/pkg/constants"
@ -12,9 +19,11 @@ import (
// Patcher defines multiple variables that need to be patched.
type Patcher struct {
labels map[string]string
annotations map[string]string
volume *operatorv1alpha1.VolumeData
labels map[string]string
annotations map[string]string
extraArgs map[string]string
featureGates map[string]bool
volume *operatorv1alpha1.VolumeData
}
// NewPatcher returns a patcher.
@ -34,6 +43,18 @@ func (p *Patcher) WithAnnotations(annotations labels.Set) *Patcher {
return p
}
// WithExtraArgs sets extraArgs to the patcher.
func (p *Patcher) WithExtraArgs(extraArgs map[string]string) *Patcher {
p.extraArgs = extraArgs
return p
}
// WithFeatureGates sets featureGates to the patcher.
func (p *Patcher) WithFeatureGates(featureGates map[string]bool) *Patcher {
p.featureGates = featureGates
return p
}
// WithVolumeData sets VolumeData to the patcher.
func (p *Patcher) WithVolumeData(volume *operatorv1alpha1.VolumeData) *Patcher {
p.volume = volume
@ -47,6 +68,33 @@ func (p *Patcher) ForDeployment(deployment *appsv1.Deployment) {
deployment.Annotations = labels.Merge(deployment.Annotations, p.annotations)
deployment.Spec.Template.Annotations = labels.Merge(deployment.Spec.Template.Annotations, p.annotations)
if len(p.extraArgs) != 0 || len(p.featureGates) != 0 {
// It's considered the first container is the karmada component by default.
baseArguments := deployment.Spec.Template.Spec.Containers[0].Command
argsMap := parseArgumentListToMap(baseArguments)
overrideArgs := map[string]string{}
// merge featureGates and build to an argurment.
if len(p.featureGates) != 0 {
baseFeatureGates := map[string]bool{}
if argument, ok := argsMap["feature-gates"]; ok {
baseFeatureGates = parseFeatrueGatesArgumentToMap(argument)
}
overrideArgs["feature-gates"] = buildFeatureGatesArgumentFromMap(baseFeatureGates, p.featureGates)
}
for key, val := range p.extraArgs {
overrideArgs[key] = val
}
// the first argument is most often the binary name
command := []string{baseArguments[0]}
command = append(command, buildArgumentListFromMap(argsMap, overrideArgs)...)
deployment.Spec.Template.Spec.Containers[0].Command = command
}
}
// ForStatefulSet patches the statefulset manifest.
@ -60,6 +108,139 @@ func (p *Patcher) ForStatefulSet(sts *appsv1.StatefulSet) {
if p.volume != nil {
patchVolumeForStatefulSet(sts, p.volume)
}
if len(p.extraArgs) != 0 {
// It's considered the first container is the karmada component by default.
baseArguments := sts.Spec.Template.Spec.Containers[0].Command
argsMap := parseArgumentListToMap(baseArguments)
sts.Spec.Template.Spec.Containers[0].Command = buildArgumentListFromMap(argsMap, p.extraArgs)
}
}
func buildArgumentListFromMap(baseArguments, overrideArguments map[string]string) []string {
var command []string
var keys []string
argsMap := make(map[string]string)
for k, v := range baseArguments {
argsMap[k] = v
}
for k, v := range overrideArguments {
argsMap[k] = v
}
for k := range argsMap {
keys = append(keys, k)
}
sort.Strings(keys)
for _, k := range keys {
command = append(command, fmt.Sprintf("--%s=%s", k, argsMap[k]))
}
return command
}
func parseFeatrueGatesArgumentToMap(featureGates string) map[string]bool {
featureGateSlice := strings.Split(featureGates, ",")
featureGatesMap := map[string]bool{}
for _, featureGate := range featureGateSlice {
key, val, err := parseFeatrueGate(featureGate)
if err != nil {
continue
}
featureGatesMap[key] = val
}
return featureGatesMap
}
func buildFeatureGatesArgumentFromMap(baseFeatureGates, overrideFeatureGates map[string]bool) string {
var featureGates []string
var keys []string
featureGateMap := make(map[string]bool)
for k, v := range baseFeatureGates {
featureGateMap[k] = v
}
for k, v := range overrideFeatureGates {
featureGateMap[k] = v
}
for k := range featureGateMap {
keys = append(keys, k)
}
sort.Strings(keys)
for _, k := range keys {
featureGates = append(featureGates, fmt.Sprintf("%s=%s", k, strconv.FormatBool(featureGateMap[k])))
}
return strings.Join(featureGates, ",")
}
func parseArgumentListToMap(arguments []string) map[string]string {
resultingMap := map[string]string{}
for i, arg := range arguments {
key, val, err := parseArgument(arg)
// Ignore if the first argument doesn't satisfy the criteria, it's most often the binary name
// Warn in all other cases, but don't error out. This can happen only if the user has edited the argument list by hand, so they might know what they are doing
if err != nil {
if i != 0 {
klog.Warningf("WARNING: The component argument %q could not be parsed correctly. The argument must be of the form %q. Skipping...\n", arg, "--")
}
continue
}
resultingMap[key] = val
}
return resultingMap
}
func parseArgument(arg string) (string, string, error) {
if !strings.HasPrefix(arg, "--") {
return "", "", errors.New("the argument should start with '--'")
}
if !strings.Contains(arg, "=") {
return "", "", errors.New("the argument should have a '=' between the flag and the value")
}
arg = strings.TrimPrefix(arg, "--")
keyvalSlice := strings.SplitN(arg, "=", 2)
if len(keyvalSlice) != 2 {
return "", "", errors.New("the argument must have both a key and a value")
}
if len(keyvalSlice[0]) == 0 {
return "", "", errors.New("the argument must have a key")
}
return keyvalSlice[0], keyvalSlice[1], nil
}
func parseFeatrueGate(featureGate string) (string, bool, error) {
if !strings.Contains(featureGate, "=") {
return "", false, errors.New("the featureGate should have a '=' between the flag and the value")
}
keyvalSlice := strings.SplitN(featureGate, "=", 2)
if len(keyvalSlice) != 2 {
return "", false, errors.New("the featureGate must have both a key and a value")
}
if len(keyvalSlice[0]) == 0 {
return "", false, errors.New("the featureGate must have a key")
}
val, err := strconv.ParseBool(keyvalSlice[1])
if err != nil {
return "", false, errors.New("the featureGate value must have a value of type bool")
}
return keyvalSlice[0], val, nil
}
func patchVolumeForStatefulSet(sts *appsv1.StatefulSet, volume *operatorv1alpha1.VolumeData) {