diff --git a/test/e2e/framework/resourceinterpretercustomization.go b/test/e2e/framework/resourceinterpretercustomization.go new file mode 100644 index 000000000..66c707a1a --- /dev/null +++ b/test/e2e/framework/resourceinterpretercustomization.go @@ -0,0 +1,29 @@ +package framework + +import ( + "context" + "fmt" + + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + configv1alpha1 "github.com/karmada-io/karmada/pkg/apis/config/v1alpha1" + karmada "github.com/karmada-io/karmada/pkg/generated/clientset/versioned" +) + +// CreateResourceInterpreterCustomization creates ResourceInterpreterCustomization with karmada client. +func CreateResourceInterpreterCustomization(client karmada.Interface, customization *configv1alpha1.ResourceInterpreterCustomization) { + ginkgo.By(fmt.Sprintf("Creating ResourceInterpreterCustomization(%s)", customization.Name), func() { + _, err := client.ConfigV1alpha1().ResourceInterpreterCustomizations().Create(context.TODO(), customization, metav1.CreateOptions{}) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + }) +} + +// DeleteResourceInterpreterCustomization deletes ResourceInterpreterCustomization with karmada client. +func DeleteResourceInterpreterCustomization(client karmada.Interface, name string) { + ginkgo.By(fmt.Sprintf("Deleting ResourceInterpreterCustomization(%s)", name), func() { + err := client.ConfigV1alpha1().ResourceInterpreterCustomizations().Delete(context.TODO(), name, metav1.DeleteOptions{}) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + }) +} diff --git a/test/e2e/resourceinterpreter_test.go b/test/e2e/resourceinterpreter_test.go index 589a06770..c327323ff 100644 --- a/test/e2e/resourceinterpreter_test.go +++ b/test/e2e/resourceinterpreter_test.go @@ -4,10 +4,14 @@ import ( "context" "encoding/json" "fmt" + "reflect" "time" "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/rand" "k8s.io/apimachinery/pkg/util/wait" @@ -15,6 +19,7 @@ import ( "k8s.io/utils/pointer" workloadv1alpha1 "github.com/karmada-io/karmada/examples/customresourceinterpreter/apis/workload/v1alpha1" + configv1alpha1 "github.com/karmada-io/karmada/pkg/apis/config/v1alpha1" policyv1alpha1 "github.com/karmada-io/karmada/pkg/apis/policy/v1alpha1" workv1alpha2 "github.com/karmada-io/karmada/pkg/apis/work/v1alpha2" "github.com/karmada-io/karmada/pkg/util/names" @@ -272,3 +277,149 @@ var _ = ginkgo.Describe("Resource interpreter webhook testing", func() { }) }) }) + +var _ = framework.SerialDescribe("Resource interpreter customization testing", func() { + var customization *configv1alpha1.ResourceInterpreterCustomization + var deployment *appsv1.Deployment + var policy *policyv1alpha1.PropagationPolicy + // We only need to test any one of the member clusters. + var targetCluster string + + ginkgo.BeforeEach(func() { + targetCluster = framework.ClusterNames()[rand.Intn(len(framework.ClusterNames()))] + deployment = testhelper.NewDeployment(testNamespace, deploymentNamePrefix+rand.String(RandomStrLength)) + policy = testhelper.NewPropagationPolicy(testNamespace, deployment.Name, []policyv1alpha1.ResourceSelector{ + { + APIVersion: deployment.APIVersion, + Kind: deployment.Kind, + Name: deployment.Name, + }, + }, policyv1alpha1.Placement{ + ClusterAffinity: &policyv1alpha1.ClusterAffinity{ + ClusterNames: []string{targetCluster}, + }, + }) + }) + + ginkgo.JustBeforeEach(func() { + framework.CreateResourceInterpreterCustomization(karmadaClient, customization) + // Wait for resource interpreter informer synced. + time.Sleep(time.Second) + + framework.CreatePropagationPolicy(karmadaClient, policy) + framework.CreateDeployment(kubeClient, deployment) + ginkgo.DeferCleanup(func() { + framework.RemoveDeployment(kubeClient, deployment.Namespace, deployment.Name) + framework.RemovePropagationPolicy(karmadaClient, policy.Namespace, policy.Name) + framework.DeleteResourceInterpreterCustomization(karmadaClient, customization.Name) + }) + }) + + ginkgo.Context("InterpreterOperation InterpretReplica testing", func() { + ginkgo.BeforeEach(func() { + customization = testhelper.NewResourceInterpreterCustomization( + "interpreter-customization"+rand.String(RandomStrLength), + configv1alpha1.CustomizationTarget{ + APIVersion: "apps/v1", + Kind: "Deployment", + }, + configv1alpha1.CustomizationRules{ + ReplicaResource: &configv1alpha1.ReplicaResourceRequirement{ + LuaScript: ` +function GetReplicas(desiredObj) + replica = desiredObj.spec.replicas + 1 + requirement = {} + requirement.nodeClaim = {} + requirement.nodeClaim.nodeSelector = desiredObj.spec.template.spec.nodeSelector + requirement.nodeClaim.tolerations = desiredObj.spec.template.spec.tolerations + requirement.resourceRequest = desiredObj.spec.template.spec.containers[1].resources.limits + return replica, requirement +end`, + }, + }) + }) + + ginkgo.It("InterpretReplica testing", func() { + ginkgo.By("check if workload's replica is interpreted", func() { + resourceBindingName := names.GenerateBindingName(deployment.Kind, deployment.Name) + // Just for the current test case to distinguish the build-in logic. + expectedReplicas := *deployment.Spec.Replicas + 1 + expectedReplicaRequirements := &workv1alpha2.ReplicaRequirements{ + ResourceRequest: map[corev1.ResourceName]resource.Quantity{ + corev1.ResourceCPU: resource.MustParse("100m"), + }} + + gomega.Eventually(func(g gomega.Gomega) (bool, error) { + resourceBinding, err := karmadaClient.WorkV1alpha2().ResourceBindings(deployment.Namespace).Get(context.TODO(), resourceBindingName, metav1.GetOptions{}) + g.Expect(err).NotTo(gomega.HaveOccurred()) + + klog.Infof(fmt.Sprintf("ResourceBinding(%s/%s)'s replicas is %d, expected: %d.", + resourceBinding.Namespace, resourceBinding.Name, resourceBinding.Spec.Replicas, expectedReplicas)) + if resourceBinding.Spec.Replicas != expectedReplicas { + return false, nil + } + + klog.Infof(fmt.Sprintf("ResourceBinding(%s/%s)'s replicaRequirements is %+v, expected: %+v.", + resourceBinding.Namespace, resourceBinding.Name, resourceBinding.Spec.ReplicaRequirements, expectedReplicaRequirements)) + return reflect.DeepEqual(resourceBinding.Spec.ReplicaRequirements, expectedReplicaRequirements), nil + }, pollTimeout, pollInterval).Should(gomega.Equal(true)) + }) + }) + }) + + ginkgo.Context("InterpreterOperation ReviseReplica testing", func() { + ginkgo.BeforeEach(func() { + customization = testhelper.NewResourceInterpreterCustomization( + "interpreter-customization"+rand.String(RandomStrLength), + configv1alpha1.CustomizationTarget{ + APIVersion: "apps/v1", + Kind: "Deployment", + }, + configv1alpha1.CustomizationRules{ + ReplicaRevision: &configv1alpha1.ReplicaRevision{ + LuaScript: ` +function ReviseReplica(obj, desiredReplica) + obj.spec.replicas = desiredReplica + 1 + return obj +end`, + }, + }) + }) + + ginkgo.BeforeEach(func() { + sumWeight := 0 + staticWeightLists := make([]policyv1alpha1.StaticClusterWeight, 0) + for index, clusterName := range framework.ClusterNames() { + staticWeightList := policyv1alpha1.StaticClusterWeight{ + TargetCluster: policyv1alpha1.ClusterAffinity{ + ClusterNames: []string{clusterName}, + }, + Weight: int64(index + 1), + } + sumWeight += index + 1 + staticWeightLists = append(staticWeightLists, staticWeightList) + } + deployment.Spec.Replicas = pointer.Int32Ptr(int32(sumWeight)) + policy.Spec.Placement = policyv1alpha1.Placement{ + ClusterAffinity: &policyv1alpha1.ClusterAffinity{ + ClusterNames: framework.ClusterNames(), + }, + ReplicaScheduling: &policyv1alpha1.ReplicaSchedulingStrategy{ + ReplicaDivisionPreference: policyv1alpha1.ReplicaDivisionPreferenceWeighted, + ReplicaSchedulingType: policyv1alpha1.ReplicaSchedulingTypeDivided, + WeightPreference: &policyv1alpha1.ClusterPreferences{ + StaticWeightList: staticWeightLists, + }, + }, + } + }) + + ginkgo.It("ReviseReplica testing", func() { + for index, clusterName := range framework.ClusterNames() { + framework.WaitDeploymentPresentOnClusterFitWith(clusterName, deployment.Namespace, deployment.Name, func(deployment *appsv1.Deployment) bool { + return *deployment.Spec.Replicas == int32(index+1)+1 + }) + } + }) + }) +}) diff --git a/test/helper/config.go b/test/helper/config.go new file mode 100644 index 000000000..9be65ae3d --- /dev/null +++ b/test/helper/config.go @@ -0,0 +1,21 @@ +package helper + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + configv1alpha1 "github.com/karmada-io/karmada/pkg/apis/config/v1alpha1" +) + +// NewResourceInterpreterCustomization will build a ResourceInterpreterCustomization object. +func NewResourceInterpreterCustomization( + name string, + target configv1alpha1.CustomizationTarget, + rules configv1alpha1.CustomizationRules) *configv1alpha1.ResourceInterpreterCustomization { + return &configv1alpha1.ResourceInterpreterCustomization{ + ObjectMeta: metav1.ObjectMeta{Name: name}, + Spec: configv1alpha1.ResourceInterpreterCustomizationSpec{ + Target: target, + Customizations: rules, + }, + } +} diff --git a/test/helper/resource.go b/test/helper/resource.go index cffc4ed94..d9c653f8d 100644 --- a/test/helper/resource.go +++ b/test/helper/resource.go @@ -56,6 +56,11 @@ func NewDeployment(namespace string, name string) *appsv1.Deployment { Containers: []corev1.Container{{ Name: "nginx", Image: "nginx:1.19.0", + Resources: corev1.ResourceRequirements{ + Limits: map[corev1.ResourceName]resource.Quantity{ + corev1.ResourceCPU: resource.MustParse("100m"), + }, + }, }}, }, },