/* Copyright 2021 The Karmada Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package taint import ( "context" "encoding/json" "fmt" "strings" "github.com/spf13/cobra" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" utilerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/strategicpatch" "k8s.io/cli-runtime/pkg/resource" cmdutil "k8s.io/kubectl/pkg/cmd/util" "k8s.io/kubectl/pkg/util/templates" clusterv1alpha1 "github.com/karmada-io/karmada/pkg/apis/cluster/v1alpha1" "github.com/karmada-io/karmada/pkg/generated/clientset/versioned/scheme" "github.com/karmada-io/karmada/pkg/karmadactl/options" "github.com/karmada-io/karmada/pkg/karmadactl/util" utilcomp "github.com/karmada-io/karmada/pkg/karmadactl/util/completion" "github.com/karmada-io/karmada/pkg/util/lifted" ) // Exported taint constant strings to mark the type of the current operation const ( MODIFIED = "modified" TAINTED = "tainted" UNTAINTED = "untainted" ) var ( taintLong = templates.LongDesc(` Update the taints on one or more clusters. * A taint consists of a key, value, and effect. As an argument here, it is expressed as key=value:effect. * The key must begin with a letter or number, and may contain letters, numbers, hyphens, dots, and underscores, up to 253 characters. * Optionally, the key can begin with a DNS subdomain prefix and a single '/', like example.com/my-app. * The value is optional. If given, it must begin with a letter or number, and may contain letters, numbers, hyphens, dots, and underscores, up to 63 characters. * The effect must be NoSchedule, PreferNoSchedule or NoExecute. * Currently taint can only apply to cluster.`) taintExample = templates.Examples(` # Update cluster 'foo' with a taint with key 'dedicated' and value 'special-user' and effect 'NoSchedule' # If a taint with that key and effect already exists, its value is replaced as specified %[1]s taint clusters foo dedicated=special-user:NoSchedule # Remove from cluster 'foo' the taint with key 'dedicated' and effect 'NoSchedule' if one exists %[1]s taint clusters foo dedicated:NoSchedule- # Remove from cluster 'foo' all the taints with key 'dedicated' %[1]s taint clusters foo dedicated- # Add to cluster 'foo' a taint with key 'bar' and no value %[1]s taint clusters foo bar:NoSchedule`) ) // NewCmdTaint defines the `taint` command that mark cluster with taints func NewCmdTaint(f util.Factory, parentCommand string) *cobra.Command { opts := CommandTaintOption{} cmd := &cobra.Command{ Use: "taint CLUSTER NAME KEY_1=VAL_1:TAINT_EFFECT_1 ... KEY_N=VAL_N:TAINT_EFFECT_N", Short: "Update the taints on one or more clusters", Long: taintLong, Example: fmt.Sprintf(taintExample, parentCommand), SilenceUsage: true, DisableFlagsInUseLine: true, ValidArgsFunction: utilcomp.SpecifiedResourceTypeAndNameCompletionFunc(f, []string{"cluster"}), RunE: func(_ *cobra.Command, args []string) error { if err := opts.Complete(f, args); err != nil { return err } if err := opts.Validate(); err != nil { return err } if err := opts.Run(f); err != nil { return err } return nil }, Annotations: map[string]string{ util.TagCommandGroup: util.GroupClusterManagement, }, } flags := cmd.Flags() options.AddKubeConfigFlags(flags) flags.BoolVar(&opts.overwrite, "overwrite", opts.overwrite, "If true, allow taints to be overwritten, otherwise reject taint updates that overwrite existing taints.") flags.BoolVar(&opts.DryRun, "dry-run", false, "Run the command in dry-run mode, without making any server requests.") utilcomp.RegisterCompletionFuncForKarmadaContextFlag(cmd) return cmd } // CommandTaintOption holds all command options for taint type CommandTaintOption struct { // DryRun tells if run the command in dry-run mode, without making any server requests. DryRun bool resources []string taintsToAdd []corev1.Taint taintsToRemove []corev1.Taint overwrite bool builder *resource.Builder } // Complete ensures that options are valid and marshals them if necessary func (o *CommandTaintOption) Complete(f cmdutil.Factory, args []string) error { taintArgs, err := o.parseTaintArgs(args) if err != nil { return err } if len(o.resources) < 1 { return fmt.Errorf("one or more resources must be specified as ") } if len(taintArgs) < 1 { return fmt.Errorf("at least one taint update is required") } if o.taintsToAdd, o.taintsToRemove, err = lifted.ParseTaints(taintArgs); err != nil { return err } o.builder = f.NewBuilder(). WithScheme(scheme.Scheme, scheme.Scheme.PrioritizedVersionsAllGroups()...). ContinueOnError().DefaultNamespace() o.builder = o.builder.ResourceNames("cluster", o.resources[1:]...) return nil } // Validate checks to the TaintOptions to see if there is sufficient information run the command func (o *CommandTaintOption) Validate() error { resourceType := strings.ToLower(o.resources[0]) if resourceType != "cluster" && resourceType != "clusters" { return fmt.Errorf("invalid resource type %s, only [\"cluster\" \"clusters\"] are supported", resourceType) } // check the format of taint args and checks removed taints aren't in the new taints list var conflictTaints []string for _, taintAdd := range o.taintsToAdd { for _, taintRemove := range o.taintsToRemove { if taintAdd.Key != taintRemove.Key { continue } if len(taintRemove.Effect) == 0 || taintAdd.Effect == taintRemove.Effect { conflictTaint := fmt.Sprintf("{\"%s\":\"%s\"}", taintRemove.Key, taintRemove.Effect) conflictTaints = append(conflictTaints, conflictTaint) } } } if len(conflictTaints) > 0 { return fmt.Errorf("can not both modify and remove the following taint(s) in the same command: %s", strings.Join(conflictTaints, ", ")) } if len(o.resources) < 2 { return fmt.Errorf("at least one resource name must be specified") } return nil } // Run set taints for the clusters func (o *CommandTaintOption) Run(f util.Factory) error { // Get control plane kube-apiserver client karmadaClient, err := f.KarmadaClientSet() if err != nil { return err } client := karmadaClient.ClusterV1alpha1().Clusters() r := o.builder.Do() if err := r.Err(); err != nil { return err } return r.Visit(func(info *resource.Info, err error) error { if err != nil { return err } obj := info.Object name, _ := info.Name, info.Namespace oldData, err := json.Marshal(obj) if err != nil { return err } operation, err := o.updateTaints(obj) if err != nil { return err } newData, err := json.Marshal(obj) if err != nil { return err } if !o.DryRun { patchBytes, patchErr := strategicpatch.CreateTwoWayMergePatch(oldData, newData, obj) cluster, err := client.Get(context.TODO(), name, metav1.GetOptions{}) if err != nil { return err } if patchErr == nil { _, err = client.Patch(context.TODO(), cluster.Name, types.MergePatchType, patchBytes, metav1.PatchOptions{}) } else { _, err = client.Update(context.TODO(), cluster, metav1.UpdateOptions{}) } if err != nil { return err } } fmt.Println("cluster/" + name + " " + operation) return nil }) } func (o *CommandTaintOption) updateTaints(obj runtime.Object) (string, error) { cluster, ok := obj.(*clusterv1alpha1.Cluster) if !ok { return "", fmt.Errorf("unexpected type %T, expected Cluster", obj) } if !o.overwrite { if exists := checkIfTaintsAlreadyExists(cluster.Spec.Taints, o.taintsToAdd); len(exists) != 0 { return "", fmt.Errorf("cluster %s already has %v taint(s) with same effect(s) and --overwrite is false", cluster.Name, exists) } } operation, newTaints, err := reorganizeTaints(cluster, o.overwrite, o.taintsToAdd, o.taintsToRemove) if err != nil { return "", err } cluster.Spec.Taints = newTaints return operation, nil } // parseTaintArgs retrieves resource and taint args from args func (o *CommandTaintOption) parseTaintArgs(args []string) ([]string, error) { var taintArgs []string metTaintArg := false for _, s := range args { isTaint := strings.Contains(s, "=") || strings.Contains(s, ":") || strings.HasSuffix(s, "-") switch { case !metTaintArg && isTaint: metTaintArg = true fallthrough case metTaintArg && isTaint: taintArgs = append(taintArgs, s) case !metTaintArg && !isTaint: o.resources = append(o.resources, s) case metTaintArg && !isTaint: return nil, fmt.Errorf("all resources must be specified before taint changes: %s", s) } } return taintArgs, nil } // reorganizeTaints returns the updated set of taints, taking into account old taints that were not updated, // old taints that were updated, old taints that were deleted, and new taints. func reorganizeTaints(cluster *clusterv1alpha1.Cluster, overwrite bool, taintsToAdd []corev1.Taint, taintsToRemove []corev1.Taint) (string, []corev1.Taint, error) { newTaints := append([]corev1.Taint{}, taintsToAdd...) oldTaints := cluster.Spec.Taints // add taints that already existing but not updated to newTaints added := addTaints(oldTaints, &newTaints) allErrs, deleted := deleteTaints(taintsToRemove, &newTaints) if (added && deleted) || overwrite { return MODIFIED, newTaints, utilerrors.NewAggregate(allErrs) } else if added { return TAINTED, newTaints, utilerrors.NewAggregate(allErrs) } return UNTAINTED, newTaints, utilerrors.NewAggregate(allErrs) } // deleteTaints deletes the given taints from the cluster's taintlist. func deleteTaints(taintsToRemove []corev1.Taint, newTaints *[]corev1.Taint) ([]error, bool) { var allErrs []error var removed bool for i, taintToRemove := range taintsToRemove { if len(taintToRemove.Effect) > 0 { *newTaints, removed = deleteTaint(*newTaints, &taintsToRemove[i]) } else { *newTaints, removed = deleteTaintsByKey(*newTaints, taintToRemove.Key) } if !removed { allErrs = append(allErrs, fmt.Errorf("taint %q not found", taintToRemove.ToString())) } } return allErrs, removed } // addTaints adds existing ones to newTaints list and updates the newTaints List. func addTaints(oldTaints []corev1.Taint, newTaints *[]corev1.Taint) bool { for i, oldTaint := range oldTaints { existsInNew := false for _, taint := range *newTaints { if taint.MatchTaint(&oldTaints[i]) { existsInNew = true break } } if !existsInNew { *newTaints = append(*newTaints, oldTaint) } } return len(oldTaints) != len(*newTaints) } // checkIfTaintsAlreadyExists checks if the cluster already has taints that we want to add and returns a string with taint keys. func checkIfTaintsAlreadyExists(oldTaints []corev1.Taint, taints []corev1.Taint) string { existingTaintList := make([]string, 0) for _, taint := range taints { for _, oldTaint := range oldTaints { if taint.Key == oldTaint.Key && taint.Effect == oldTaint.Effect { existingTaintList = append(existingTaintList, taint.Key) } } } return strings.Join(existingTaintList, ",") } // deleteTaintsByKey removes all the taints that have the same key to given taintKey func deleteTaintsByKey(taints []corev1.Taint, taintKey string) ([]corev1.Taint, bool) { var newTaints []corev1.Taint for i := range taints { if taintKey == taints[i].Key { continue } newTaints = append(newTaints, taints[i]) } return newTaints, len(taints) != len(newTaints) } // deleteTaint removes all the taints that have the same key and effect to given taintToDelete. func deleteTaint(taints []corev1.Taint, taintToDelete *corev1.Taint) ([]corev1.Taint, bool) { var newTaints []corev1.Taint for i := range taints { if taintToDelete.MatchTaint(&taints[i]) { continue } newTaints = append(newTaints, taints[i]) } return newTaints, len(taints) != len(newTaints) }