transform hpa to propagationwork (#147)

Signed-off-by: chenxianpao <chenxianpao@huawei.com>

cmd/controller-manager/app/controllermanager.go

@@ -104,7 +104,9 @@ func setupControllers(mgr controllerruntime.Manager, stopChan <-chan struct{}) {
 
 	hpaController := &hpa.HorizontalPodAutoscalerController{
 		Client:        mgr.GetClient(),
+		DynamicClient: dynamicClientSet,
 		EventRecorder: mgr.GetEventRecorderFor(hpa.ControllerName),
+		RESTMapper:    mgr.GetRESTMapper(),
 	}
 	if err := hpaController.SetupWithManager(mgr); err != nil {
 		klog.Fatalf("Failed to setup hpa controller: %v", err)

pkg/controllers/hpa/hpa_controller.go

@@ -5,10 +5,22 @@ import (
 
 	autoscalingv1 "k8s.io/api/autoscaling/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/api/meta"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/dynamic"
 	"k8s.io/client-go/tools/record"
 	"k8s.io/klog/v2"
 	controllerruntime "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	"github.com/karmada-io/karmada/pkg/apis/policy/v1alpha1"
+	"github.com/karmada-io/karmada/pkg/util"
+	"github.com/karmada-io/karmada/pkg/util/names"
+	"github.com/karmada-io/karmada/pkg/util/restmapper"
 )
 
 // ControllerName is the controller name that will be used when reporting events.
@@ -16,8 +28,10 @@ const ControllerName = "hpa-controller"
 
 // HorizontalPodAutoscalerController is to sync HorizontalPodAutoscaler.
 type HorizontalPodAutoscalerController struct {
-	client.Client // used to operate HorizontalPodAutoscaler resources.
+	client.Client                   // used to operate HorizontalPodAutoscaler resources.
+	DynamicClient dynamic.Interface // used to fetch arbitrary resources.
 	EventRecorder record.EventRecorder
+	RESTMapper    meta.RESTMapper
 }
 
 // Reconcile performs a full reconciliation for the object referred to by the Request.
@@ -41,9 +55,94 @@ func (c *HorizontalPodAutoscalerController) Reconcile(req controllerruntime.Requ
 		return controllerruntime.Result{}, nil
 	}
 
+	return c.syncHPA(hpa)
+}
+
+// syncHPA gets placement from propagationBinding according to targetRef in hpa, then builds propagationWorks in target execution namespaces.
+func (c *HorizontalPodAutoscalerController) syncHPA(hpa *autoscalingv1.HorizontalPodAutoscaler) (controllerruntime.Result, error) {
+	clusters, err := c.getTargetPlacement(hpa.Spec.ScaleTargetRef, hpa.GetNamespace())
+	if err != nil {
+		klog.Errorf("Failed to get target placement by hpa %s/%s. Error: %v.", hpa.GetNamespace(), hpa.GetName(), err)
+		return controllerruntime.Result{Requeue: true}, err
+	}
+	err = c.buildPropagationWorks(hpa, clusters)
+	if err != nil {
+		klog.Errorf("Failed to build propagationWork for hpa %s/%s. Error: %v.", hpa.GetNamespace(), hpa.GetName(), err)
+		return controllerruntime.Result{Requeue: true}, err
+	}
 	return controllerruntime.Result{}, nil
 }
 
+// buildPropagationWorks transforms hpa obj to unstructured, creates or updates propagationWorks in the target execution namespaces.
+func (c *HorizontalPodAutoscalerController) buildPropagationWorks(hpa *autoscalingv1.HorizontalPodAutoscaler, clusters []string) error {
+	uncastObj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(hpa)
+	if err != nil {
+		klog.Errorf("Failed to transform hpa %s/%s. Error: %v", hpa.GetNamespace(), hpa.GetName(), err)
+		return nil
+	}
+	hpaObj := &unstructured.Unstructured{Object: uncastObj}
+	util.RemoveIrrelevantField(hpaObj)
+	for _, clusterName := range clusters {
+		hpaJSON, err := hpaObj.MarshalJSON()
+		if err != nil {
+			klog.Errorf("Failed to marshal hpa %s/%s. Error: %v",
+				hpaObj.GetNamespace(), hpaObj.GetName(), err)
+			return err
+		}
+
+		executionSpace, err := names.GenerateExecutionSpaceName(clusterName)
+		if err != nil {
+			klog.Errorf("Failed to ensure PropagationWork for cluster: %s. Error: %v.", clusterName, err)
+			return err
+		}
+
+		hpaName := names.GenerateBindingName(hpaObj.GetNamespace(), hpaObj.GetKind(), hpaObj.GetName())
+		objectMeta := metav1.ObjectMeta{
+			Name:       hpaName,
+			Namespace:  executionSpace,
+			Finalizers: []string{util.ExecutionControllerFinalizer},
+			OwnerReferences: []metav1.OwnerReference{
+				*metav1.NewControllerRef(hpa, hpa.GroupVersionKind()),
+			},
+			Labels: map[string]string{util.OwnerLabel: names.GenerateOwnerLabelValue(hpa.GetNamespace(), hpa.GetName())},
+		}
+
+		err = util.CreateOrUpdatePropagationWork(c.Client, objectMeta, hpaJSON)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// getTargetPlacement gets target clusters by CrossVersionObjectReference in hpa object. We can find
+// the propagationBinding by resource with special naming rule, then get target clusters from propagationBinding.
+func (c *HorizontalPodAutoscalerController) getTargetPlacement(objRef autoscalingv1.CrossVersionObjectReference, namespace string) ([]string, error) {
+	// according to targetRef, find the resource.
+	dynamicResource, err := restmapper.GetGroupVersionResource(c.RESTMapper,
+		schema.FromAPIVersionAndKind(objRef.APIVersion, objRef.Kind))
+	if err != nil {
+		klog.Errorf("Failed to get GVR from GVK %s %s. Error: %v", objRef.APIVersion, objRef.Kind, err)
+		return nil, err
+	}
+
+	// Kind in CrossVersionObjectReference is not equal to the kind in bindingName, need to get obj from cache.
+	unstructuredWorkLoad, err := c.DynamicClient.Resource(dynamicResource).Namespace(namespace).Get(context.TODO(), objRef.Name, metav1.GetOptions{})
+	if err != nil {
+		return nil, err
+	}
+	bindingName := names.GenerateBindingName(unstructuredWorkLoad.GetNamespace(), unstructuredWorkLoad.GetKind(), unstructuredWorkLoad.GetName())
+	binding := &v1alpha1.PropagationBinding{}
+	namespacedName := types.NamespacedName{
+		Namespace: namespace,
+		Name:      bindingName,
+	}
+	if err := c.Client.Get(context.TODO(), namespacedName, binding); err != nil {
+		return nil, err
+	}
+	return util.GetBindingClusterNames(binding), nil
+}
+
 // SetupWithManager creates a controller and register to controller manager.
 func (c *HorizontalPodAutoscalerController) SetupWithManager(mgr controllerruntime.Manager) error {
 	return controllerruntime.NewControllerManagedBy(mgr).For(&autoscalingv1.HorizontalPodAutoscaler{}).Complete(c)

pkg/util/binding.go

@@ -0,0 +1,59 @@
+package util
+
+import (
+	"context"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/klog/v2"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+
+	"github.com/karmada-io/karmada/pkg/apis/policy/v1alpha1"
+)
+
+// GetBindingClusterNames will get clusterName list from bind clusters field
+func GetBindingClusterNames(binding *v1alpha1.PropagationBinding) []string {
+	var clusterNames []string
+	for _, targetCluster := range binding.Spec.Clusters {
+		clusterNames = append(clusterNames, targetCluster.Name)
+	}
+	return clusterNames
+}
+
+// CreateOrUpdatePropagationWork creates or updates propagationWork by controllerutil.CreateOrUpdate
+func CreateOrUpdatePropagationWork(client client.Client, objectMeta metav1.ObjectMeta, rawExtension []byte) error {
+	propagationWork := &v1alpha1.PropagationWork{
+		ObjectMeta: objectMeta,
+		Spec: v1alpha1.PropagationWorkSpec{
+			Workload: v1alpha1.WorkloadTemplate{
+				Manifests: []v1alpha1.Manifest{
+					{
+						RawExtension: runtime.RawExtension{
+							Raw: rawExtension,
+						},
+					},
+				},
+			},
+		},
+	}
+
+	runtimeObject := propagationWork.DeepCopy()
+	operationResult, err := controllerutil.CreateOrUpdate(context.TODO(), client, runtimeObject, func() error {
+		runtimeObject.Spec = propagationWork.Spec
+		return nil
+	})
+	if err != nil {
+		klog.Errorf("Failed to create/update propagationWork %s/%s. Error: %v", propagationWork.GetNamespace(), propagationWork.GetName(), err)
+		return err
+	}
+
+	if operationResult == controllerutil.OperationResultCreated {
+		klog.Infof("Create propagationWork %s/%s successfully.", propagationWork.GetNamespace(), propagationWork.GetName())
+	} else if operationResult == controllerutil.OperationResultUpdated {
+		klog.Infof("Update propagationWork %s/%s successfully.", propagationWork.GetNamespace(), propagationWork.GetName())
+	} else {
+		klog.V(2).Infof("PropagationWork %s/%s is up to date.", propagationWork.GetNamespace(), propagationWork.GetName())
+	}
+	return nil
+}

pkg/util/constants.go

@@ -8,6 +8,7 @@ const (
 	// We can use labelSelector to find who created it quickly.
 	// example1: set it in propagationBinding, the label value is propagationPolicy.
 	// example2: set it in propagationWork, the label value is propagationBinding.
+	// example3: set it in propagationWork, the label value is HPA.
 	OwnerLabel = "karmada.io/created-by"
 	// OverrideClaimKey will set in propagationwork resource, indicates that
 	// the resource is overridden by override policies

pkg/util/format.go

@@ -0,0 +1,16 @@
+package util
+
+import (
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+)
+
+// RemoveIrrelevantField will delete irrelevant field from workload. such as uid, timestamp, status
+func RemoveIrrelevantField(workload *unstructured.Unstructured) {
+	unstructured.RemoveNestedField(workload.Object, "metadata", "creationTimestamp")
+	unstructured.RemoveNestedField(workload.Object, "metadata", "generation")
+	unstructured.RemoveNestedField(workload.Object, "metadata", "resourceVersion")
+	unstructured.RemoveNestedField(workload.Object, "metadata", "selfLink")
+	unstructured.RemoveNestedField(workload.Object, "metadata", "managedFields")
+	unstructured.RemoveNestedField(workload.Object, "metadata", "uid")
+	unstructured.RemoveNestedField(workload.Object, "status")
+}