package karmadactl import ( "context" "fmt" "os" "github.com/spf13/cobra" "github.com/spf13/pflag" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/cli-runtime/pkg/genericclioptions" "k8s.io/cli-runtime/pkg/printers" "k8s.io/cli-runtime/pkg/resource" "k8s.io/client-go/dynamic" "k8s.io/client-go/rest" cmdutil "k8s.io/kubectl/pkg/cmd/util" "k8s.io/kubectl/pkg/util/templates" policyv1alpha1 "github.com/karmada-io/karmada/pkg/apis/policy/v1alpha1" workv1alpha2 "github.com/karmada-io/karmada/pkg/apis/work/v1alpha2" karmadaclientset "github.com/karmada-io/karmada/pkg/generated/clientset/versioned" "github.com/karmada-io/karmada/pkg/karmadactl/options" "github.com/karmada-io/karmada/pkg/karmadactl/util" "github.com/karmada-io/karmada/pkg/resourceinterpreter/defaultinterpreter/prune" "github.com/karmada-io/karmada/pkg/util/gclient" "github.com/karmada-io/karmada/pkg/util/names" "github.com/karmada-io/karmada/pkg/util/restmapper" ) var ( promoteShort = `Promote resources from legacy clusters to karmada control plane` promoteLong = `Promote resources from legacy clusters to karmada control plane. Requires the cluster be joined or registered.` promoteExample = templates.Examples(` # Promote deployment(default/nginx) from cluster1 to Karmada %[1]s promote deployment nginx -n default -C cluster1 # Promote deployment(default/nginx) with gvk from cluster1 to Karmada %[1]s promote deployment.v1.apps nginx -n default -C cluster1 # Dumps the artifacts but does not deploy them to Karmada, same as 'dry run' %[1]s promote deployment nginx -n default -C cluster1 -o yaml|json # Promote secret(default/default-token) from cluster1 to Karmada %[1]s promote secret default-token -n default -C cluster1 # Support to use '--cluster-kubeconfig' to specify the configuration of member cluster %[1]s promote deployment nginx -n default -C cluster1 --cluster-kubeconfig= # Support to use '--cluster-kubeconfig' and '--cluster-context' to specify the configuration of member cluster %[1]s promote deployment nginx -n default -C cluster1 --cluster-kubeconfig= --cluster-context=`) ) // NewCmdPromote defines the `promote` command that promote resources from legacy clusters func NewCmdPromote(f util.Factory, parentCommand string) *cobra.Command { opts := CommandPromoteOption{} opts.JSONYamlPrintFlags = genericclioptions.NewJSONYamlPrintFlags() cmd := &cobra.Command{ Use: "promote -n -C ", Short: promoteShort, Long: promoteLong, Example: fmt.Sprintf(promoteExample, parentCommand), SilenceUsage: true, DisableFlagsInUseLine: true, RunE: func(cmd *cobra.Command, args []string) error { if err := opts.Complete(f, args); err != nil { return err } if err := opts.Validate(); err != nil { return err } if err := opts.Run(f, args); err != nil { return err } return nil }, Annotations: map[string]string{ util.TagCommandGroup: util.GroupAdvancedCommands, }, } flag := cmd.Flags() opts.AddFlags(flag) options.AddKubeConfigFlags(flag) return cmd } // CommandPromoteOption holds all command options for promote type CommandPromoteOption struct { // Cluster is the name of legacy cluster Cluster string // Namespace is the namespace of legacy resource Namespace string // ClusterContext is the cluster's context that we are going to join with. ClusterContext string // ClusterKubeConfig is the cluster's kubeconfig path. ClusterKubeConfig string // DryRun tells if run the command in dry-run mode, without making any server requests. DryRun bool resource.FilenameOptions JSONYamlPrintFlags *genericclioptions.JSONYamlPrintFlags OutputFormat string Printer func(*meta.RESTMapping, *bool, bool, bool) (printers.ResourcePrinterFunc, error) TemplateFlags *genericclioptions.KubeTemplatePrintFlags name string gvk schema.GroupVersionKind } // AddFlags adds flags to the specified FlagSet. func (o *CommandPromoteOption) AddFlags(flags *pflag.FlagSet) { flags.StringVarP(&o.OutputFormat, "output", "o", "", "Output format. One of: json|yaml") flags.StringVarP(&o.Namespace, "namespace", "n", o.Namespace, "If present, the namespace scope for this CLI request") flags.StringVarP(&o.Cluster, "cluster", "C", "", "the name of legacy cluster (eg -C=member1)") flags.StringVar(&o.ClusterContext, "cluster-context", "", "Context name of legacy cluster in kubeconfig. Only works when there are multiple contexts in the kubeconfig.") flags.StringVar(&o.ClusterKubeConfig, "cluster-kubeconfig", "", "Path of the legacy cluster's kubeconfig.") flags.BoolVar(&o.DryRun, "dry-run", false, "Run the command in dry-run mode, without making any server requests.") } // Complete ensures that options are valid and marshals them if necessary func (o *CommandPromoteOption) Complete(f util.Factory, args []string) error { var err error if len(args) != 2 { return fmt.Errorf("incorrect command format, please use correct command format") } o.name = args[1] if o.OutputFormat == "yaml" || o.OutputFormat == "json" { o.Printer = func(mapping *meta.RESTMapping, outputObjects *bool, withNamespace bool, withKind bool) (printers.ResourcePrinterFunc, error) { printer, err := o.JSONYamlPrintFlags.ToPrinter(o.OutputFormat) if genericclioptions.IsNoCompatiblePrinterError(err) { return nil, err } printer, err = printers.NewTypeSetter(gclient.NewSchema()).WrapToPrinter(printer, nil) if err != nil { return nil, err } return printer.PrintObj, nil } } if o.Namespace == "" { o.Namespace, _, err = f.ToRawKubeConfigLoader().Namespace() if err != nil { return fmt.Errorf("failed to get namespace from Factory. error: %w", err) } } // If '--cluster-context' not specified, take the cluster name as the context. if len(o.ClusterContext) == 0 { o.ClusterContext = o.Cluster } return nil } // Validate checks to the PromoteOptions to see if there is sufficient information run the command func (o *CommandPromoteOption) Validate() error { if o.Cluster == "" { return fmt.Errorf("the cluster cannot be empty") } if o.OutputFormat != "" && o.OutputFormat != "yaml" && o.OutputFormat != "json" { return fmt.Errorf("output format is only one of json and yaml") } return nil } // Run promote resources from legacy clusters func (o *CommandPromoteOption) Run(f util.Factory, args []string) error { var memberClusterFactory cmdutil.Factory var err error if o.ClusterKubeConfig != "" { kubeConfigFlags := genericclioptions.NewConfigFlags(true).WithDeprecatedPasswordFlag() kubeConfigFlags.KubeConfig = &o.ClusterKubeConfig kubeConfigFlags.Context = &o.ClusterContext memberClusterFactory = cmdutil.NewFactory(kubeConfigFlags) } else { memberClusterFactory, err = f.FactoryForMemberCluster(o.Cluster) if err != nil { return fmt.Errorf("failed to get Factory of the member cluster. err: %w", err) } } objInfo, err := o.getObjInfo(memberClusterFactory, o.Cluster, args) if err != nil { return fmt.Errorf("failed to get resource in cluster(%s). err: %w", o.Cluster, err) } obj := objInfo.Info.Object.(*unstructured.Unstructured) o.gvk = obj.GetObjectKind().GroupVersionKind() controlPlaneRestConfig, err := f.ToRESTConfig() if err != nil { return fmt.Errorf("failed to get control plane rest config. err: %w", err) } mapper, err := restmapper.MapperProvider(controlPlaneRestConfig) if err != nil { return fmt.Errorf("failed to create restmapper: %v", err) } gvr, err := restmapper.GetGroupVersionResource(mapper, o.gvk) if err != nil { return fmt.Errorf("failed to get gvr from %q: %v", o.gvk, err) } return o.promote(controlPlaneRestConfig, obj, gvr) } func (o *CommandPromoteOption) promote(controlPlaneRestConfig *rest.Config, obj *unstructured.Unstructured, gvr schema.GroupVersionResource) error { if err := preprocessResource(obj); err != nil { return fmt.Errorf("failed to preprocess resource %q(%s/%s) in control plane: %v", gvr, o.Namespace, o.name, err) } if o.OutputFormat != "" { // only print the resource template and Policy err := o.printObjectAndPolicy(obj, gvr) return err } if o.DryRun { return nil } controlPlaneDynamicClient := dynamic.NewForConfigOrDie(controlPlaneRestConfig) karmadaClient := karmadaclientset.NewForConfigOrDie(controlPlaneRestConfig) if len(obj.GetNamespace()) == 0 { _, err := controlPlaneDynamicClient.Resource(gvr).Get(context.TODO(), o.name, metav1.GetOptions{}) if err == nil { fmt.Printf("Resource %q(%s) already exist in karmada control plane, you can edit PropagationPolicy and OverridePolicy to propagate it\n", gvr, o.name) return nil } if !apierrors.IsNotFound(err) { return fmt.Errorf("failed to get resource %q(%s) in control plane: %v", gvr, o.name, err) } _, err = controlPlaneDynamicClient.Resource(gvr).Create(context.TODO(), obj, metav1.CreateOptions{}) if err != nil { return fmt.Errorf("failed to create resource %q(%s) in control plane: %v", gvr, o.name, err) } err = o.createClusterPropagationPolicy(karmadaClient, gvr) if err != nil { return err } fmt.Printf("Resource %q(%s) is promoted successfully\n", gvr, o.name) } else { _, err := controlPlaneDynamicClient.Resource(gvr).Namespace(o.Namespace).Get(context.TODO(), o.name, metav1.GetOptions{}) if err == nil { fmt.Printf("Resource %q(%s/%s) already exist in karmada control plane, you can edit PropagationPolicy and OverridePolicy to propagate it\n", gvr, o.Namespace, o.name) return nil } if !apierrors.IsNotFound(err) { return fmt.Errorf("failed to get resource %q(%s/%s) in control plane: %v", gvr, o.Namespace, o.name, err) } _, err = controlPlaneDynamicClient.Resource(gvr).Namespace(o.Namespace).Create(context.TODO(), obj, metav1.CreateOptions{}) if err != nil { return fmt.Errorf("failed to create resource %q(%s/%s) in control plane: %v", gvr, o.Namespace, o.name, err) } err = o.createPropagationPolicy(karmadaClient, gvr) if err != nil { return err } fmt.Printf("Resource %q(%s/%s) is promoted successfully\n", gvr, o.Namespace, o.name) } return nil } // getObjInfo get obj info in member cluster func (o *CommandPromoteOption) getObjInfo(f cmdutil.Factory, cluster string, args []string) (*Obj, error) { r := f.NewBuilder(). Unstructured(). NamespaceParam(o.Namespace). FilenameParam(false, &o.FilenameOptions). RequestChunksOf(500). ResourceTypeOrNameArgs(true, args...). ContinueOnError(). Latest(). Flatten(). Do() r.IgnoreErrors(apierrors.IsNotFound) infos, err := r.Infos() if err != nil { return nil, err } if len(infos) == 0 { return nil, fmt.Errorf("the %s or %s don't exist in cluster(%s)", args[0], args[1], o.Cluster) } obj := &Obj{ Cluster: cluster, Info: infos[0], } return obj, nil } // printObjectAndPolicy print the converted resource func (o *CommandPromoteOption) printObjectAndPolicy(obj *unstructured.Unstructured, gvr schema.GroupVersionResource) error { printer, err := o.Printer(nil, nil, false, false) if err != nil { return fmt.Errorf("failed to initialize k8s printer. err: %v", err) } if err = printer.PrintObj(obj, os.Stdout); err != nil { return fmt.Errorf("failed to print the resource template. err: %v", err) } if len(obj.GetNamespace()) == 0 { cpp := buildClusterPropagationPolicy(o.name, o.Cluster, gvr, o.gvk) if err = printer.PrintObj(cpp, os.Stdout); err != nil { return fmt.Errorf("failed to print the ClusterPropagationPolicy. err: %v", err) } } else { pp := buildPropagationPolicy(o.name, o.Namespace, o.Cluster, gvr, o.gvk) if err = printer.PrintObj(pp, os.Stdout); err != nil { return fmt.Errorf("failed to print the PropagationPolicy. err: %v", err) } } return nil } // createPropagationPolicy create PropagationPolicy in karmada control plane func (o *CommandPromoteOption) createPropagationPolicy(karmadaClient *karmadaclientset.Clientset, gvr schema.GroupVersionResource) error { name := names.GeneratePolicyName(o.Namespace, o.name, o.gvk.String()) _, err := karmadaClient.PolicyV1alpha1().PropagationPolicies(o.Namespace).Get(context.TODO(), name, metav1.GetOptions{}) if err != nil && apierrors.IsNotFound(err) { pp := buildPropagationPolicy(o.name, o.Namespace, o.Cluster, gvr, o.gvk) _, err = karmadaClient.PolicyV1alpha1().PropagationPolicies(o.Namespace).Create(context.TODO(), pp, metav1.CreateOptions{}) return err } if err != nil { return fmt.Errorf("failed to get PropagationPolicy(%s/%s) in control plane: %v", o.Namespace, name, err) } // PropagationPolicy already exists, not to create it return fmt.Errorf("the PropagationPolicy(%s/%s) already exist, please edit it to propagate resource", o.Namespace, name) } // createClusterPropagationPolicy create ClusterPropagationPolicy in karmada control plane func (o *CommandPromoteOption) createClusterPropagationPolicy(karmadaClient *karmadaclientset.Clientset, gvr schema.GroupVersionResource) error { name := names.GeneratePolicyName("", o.name, o.gvk.String()) _, err := karmadaClient.PolicyV1alpha1().ClusterPropagationPolicies().Get(context.TODO(), name, metav1.GetOptions{}) if err != nil && apierrors.IsNotFound(err) { cpp := buildClusterPropagationPolicy(o.name, o.Cluster, gvr, o.gvk) _, err = karmadaClient.PolicyV1alpha1().ClusterPropagationPolicies().Create(context.TODO(), cpp, metav1.CreateOptions{}) return err } if err != nil { return fmt.Errorf("failed to get ClusterPropagationPolicy(%s) in control plane: %v", name, err) } // ClusterPropagationPolicy already exists, not to create it return fmt.Errorf("the ClusterPropagationPolicy(%s) already exist, please edit it to propagate resource", name) } // preprocessResource delete redundant fields to convert resource as template func preprocessResource(obj *unstructured.Unstructured) error { // remove fields that generated by kube-apiserver and no need(or can't) propagate to member clusters. if err := prune.RemoveIrrelevantField(obj); err != nil { return err } addOverwriteAnnotation(obj) return nil } // addOverwriteAnnotation add annotation "work.karmada.io/conflict-resolution" to the resource func addOverwriteAnnotation(obj *unstructured.Unstructured) { annotations := obj.DeepCopy().GetAnnotations() if annotations == nil { annotations = map[string]string{} } if _, exist := annotations[workv1alpha2.ResourceConflictResolutionAnnotation]; !exist { annotations[workv1alpha2.ResourceConflictResolutionAnnotation] = workv1alpha2.ResourceConflictResolutionOverwrite obj.SetAnnotations(annotations) } } // buildPropagationPolicy build PropagationPolicy according to resource and cluster func buildPropagationPolicy(resourceName, namespace, cluster string, gvr schema.GroupVersionResource, gvk schema.GroupVersionKind) *policyv1alpha1.PropagationPolicy { policyName := names.GeneratePolicyName(namespace, resourceName, gvk.String()) pp := &policyv1alpha1.PropagationPolicy{ ObjectMeta: metav1.ObjectMeta{ Name: policyName, Namespace: namespace, }, Spec: policyv1alpha1.PropagationSpec{ ResourceSelectors: []policyv1alpha1.ResourceSelector{ { APIVersion: gvr.GroupVersion().String(), Kind: gvk.Kind, Name: resourceName, }, }, Placement: policyv1alpha1.Placement{ ClusterAffinity: &policyv1alpha1.ClusterAffinity{ ClusterNames: []string{cluster}, }, }, }, } return pp } // buildClusterPropagationPolicy build ClusterPropagationPolicy according to resource and cluster func buildClusterPropagationPolicy(resourceName, cluster string, gvr schema.GroupVersionResource, gvk schema.GroupVersionKind) *policyv1alpha1.ClusterPropagationPolicy { policyName := names.GeneratePolicyName("", resourceName, gvk.String()) cpp := &policyv1alpha1.ClusterPropagationPolicy{ ObjectMeta: metav1.ObjectMeta{ Name: policyName, }, Spec: policyv1alpha1.PropagationSpec{ ResourceSelectors: []policyv1alpha1.ResourceSelector{ { APIVersion: gvr.GroupVersion().String(), Kind: gvk.Kind, Name: resourceName, }, }, Placement: policyv1alpha1.Placement{ ClusterAffinity: &policyv1alpha1.ClusterAffinity{ ClusterNames: []string{cluster}, }, }, }, } return cpp }