karmada/pkg/controllers/cluster/taint_manager_test.go

537 lines
19 KiB
Go

/*
Copyright 2024 The Karmada Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cluster
import (
"context"
"reflect"
"testing"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
clusterv1alpha1 "github.com/karmada-io/karmada/pkg/apis/cluster/v1alpha1"
policyv1alpha1 "github.com/karmada-io/karmada/pkg/apis/policy/v1alpha1"
workv1alpha2 "github.com/karmada-io/karmada/pkg/apis/work/v1alpha2"
"github.com/karmada-io/karmada/pkg/util"
"github.com/karmada-io/karmada/pkg/util/fedinformer/keys"
"github.com/karmada-io/karmada/pkg/util/gclient"
)
func newNoExecuteTaintManager() *NoExecuteTaintManager {
rbIndexerFunc := func(obj client.Object) []string {
rb, ok := obj.(*workv1alpha2.ResourceBinding)
if !ok {
return nil
}
return util.GetBindingClusterNames(&rb.Spec)
}
crbIndexerFunc := func(obj client.Object) []string {
crb, ok := obj.(*workv1alpha2.ClusterResourceBinding)
if !ok {
return nil
}
return util.GetBindingClusterNames(&crb.Spec)
}
mgr := &NoExecuteTaintManager{
Client: fake.NewClientBuilder().WithScheme(gclient.NewSchema()).
WithIndex(&workv1alpha2.ResourceBinding{}, rbClusterKeyIndex, rbIndexerFunc).
WithIndex(&workv1alpha2.ClusterResourceBinding{}, crbClusterKeyIndex, crbIndexerFunc).Build(),
}
bindingEvictionWorkerOptions := util.Options{
Name: "binding-eviction",
KeyFunc: nil,
ReconcileFunc: mgr.syncBindingEviction,
}
mgr.bindingEvictionWorker = util.NewAsyncWorker(bindingEvictionWorkerOptions)
clusterBindingEvictionWorkerOptions := util.Options{
Name: "cluster-binding-eviction",
KeyFunc: nil,
ReconcileFunc: mgr.syncClusterBindingEviction,
}
mgr.clusterBindingEvictionWorker = util.NewAsyncWorker(clusterBindingEvictionWorkerOptions)
return mgr
}
func TestNoExecuteTaintManager_Reconcile(t *testing.T) {
tests := []struct {
name string
cluster *clusterv1alpha1.Cluster
want reconcile.Result
wantErr bool
}{
{
name: "no taints",
cluster: &clusterv1alpha1.Cluster{
ObjectMeta: metav1.ObjectMeta{
Name: "test-cluster",
},
Spec: clusterv1alpha1.ClusterSpec{},
},
want: reconcile.Result{},
wantErr: false,
},
{
name: "have taints",
cluster: &clusterv1alpha1.Cluster{
ObjectMeta: metav1.ObjectMeta{
Name: "test-cluster",
},
Spec: clusterv1alpha1.ClusterSpec{
Taints: []corev1.Taint{
{
Key: "test-taint",
Value: "test-value",
Effect: corev1.TaintEffectNoExecute,
},
},
},
},
want: reconcile.Result{},
wantErr: false,
},
{
name: "cluster not found",
want: reconcile.Result{},
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
tc := newNoExecuteTaintManager()
if err := tc.Client.Create(context.Background(), &workv1alpha2.ResourceBinding{
TypeMeta: metav1.TypeMeta{
Kind: "ResourceBinding",
APIVersion: "work.karmada.io/v1alpha2",
},
ObjectMeta: metav1.ObjectMeta{
Name: "test-rb",
Namespace: "default",
},
Spec: workv1alpha2.ResourceBindingSpec{
Clusters: []workv1alpha2.TargetCluster{
{
Name: "test-cluster",
Replicas: 1,
},
},
},
}, &client.CreateOptions{}); err != nil {
t.Fatalf("failed to create rb, %v", err)
}
if err := tc.Client.Create(context.Background(), &workv1alpha2.ClusterResourceBinding{
TypeMeta: metav1.TypeMeta{
Kind: "ClusterResourceBinding",
APIVersion: "work.karmada.io/v1alpha2",
},
ObjectMeta: metav1.ObjectMeta{
Name: "test-crb",
},
Spec: workv1alpha2.ResourceBindingSpec{
Clusters: []workv1alpha2.TargetCluster{
{
Name: "test-cluster",
Replicas: 1,
},
},
},
}, &client.CreateOptions{}); err != nil {
t.Fatalf("failed to create crb, %v", err)
}
if tt.cluster != nil {
if err := tc.Client.Create(context.Background(), tt.cluster, &client.CreateOptions{}); err != nil {
t.Fatal(err)
return
}
}
req := reconcile.Request{NamespacedName: types.NamespacedName{Name: "test-cluster"}}
got, err := tc.Reconcile(context.Background(), req)
if (err != nil) != tt.wantErr {
t.Errorf("NoExecuteTaintManager.Reconcile() error = %v, wantErr %v", err, tt.wantErr)
return
}
if !reflect.DeepEqual(got, tt.want) {
t.Errorf("NoExecuteTaintManager.Reconcile() = %v, want %v", got, tt.want)
}
})
}
}
func TestNoExecuteTaintManager_syncBindingEviction(t *testing.T) {
replica := int32(1)
tests := []struct {
name string
rb *workv1alpha2.ResourceBinding
cluster *clusterv1alpha1.Cluster
wrb *workv1alpha2.ResourceBinding
wantErr bool
}{
{
name: "rb without tolerations",
rb: &workv1alpha2.ResourceBinding{
TypeMeta: metav1.TypeMeta{
Kind: "ResourceBinding",
APIVersion: "work.karmada.io/v1alpha2",
},
ObjectMeta: metav1.ObjectMeta{
Name: "test-rb",
Namespace: "default",
Annotations: map[string]string{"policy.karmada.io/applied-placement": `{"clusterAffinity":{"clusterNames":["member1","member2"]},"clusterTolerations":[{"key":"cluster.karmada.io/unreachable","operator":"Exists","effect":"NoExecute","tolerationSeconds":30}],"replicaScheduling":{"replicaSchedulingType":"Divided","replicaDivisionPreference":"Weighted","weightPreference":{"staticWeightList":[{"targetCluster":{"clusterNames":["member1"]},"weight":1},{"targetCluster":{"clusterNames":["member2"]},"weight":1}]}}}`},
},
Spec: workv1alpha2.ResourceBindingSpec{
Clusters: []workv1alpha2.TargetCluster{
{
Name: "test-cluster",
Replicas: 1,
},
},
},
},
cluster: &clusterv1alpha1.Cluster{
ObjectMeta: metav1.ObjectMeta{
Name: "test-cluster",
},
Spec: clusterv1alpha1.ClusterSpec{
Taints: []corev1.Taint{
{
Key: "cluster.karmada.io/not-ready",
Effect: corev1.TaintEffectNoExecute,
},
},
},
},
wrb: &workv1alpha2.ResourceBinding{
TypeMeta: metav1.TypeMeta{
Kind: "ResourceBinding",
APIVersion: "work.karmada.io/v1alpha2",
},
ObjectMeta: metav1.ObjectMeta{
Name: "test-rb",
Namespace: "default",
Annotations: map[string]string{"policy.karmada.io/applied-placement": `{"clusterAffinity":{"clusterNames":["member1","member2"]},"clusterTolerations":[{"key":"cluster.karmada.io/unreachable","operator":"Exists","effect":"NoExecute","tolerationSeconds":30}],"replicaScheduling":{"replicaSchedulingType":"Divided","replicaDivisionPreference":"Weighted","weightPreference":{"staticWeightList":[{"targetCluster":{"clusterNames":["member1"]},"weight":1},{"targetCluster":{"clusterNames":["member2"]},"weight":1}]}}}`},
},
Spec: workv1alpha2.ResourceBindingSpec{
GracefulEvictionTasks: []workv1alpha2.GracefulEvictionTask{
{
FromCluster: "test-cluster",
PurgeMode: policyv1alpha1.Graciously,
Replicas: &replica,
Reason: workv1alpha2.EvictionReasonTaintUntolerated,
Producer: workv1alpha2.EvictionProducerTaintManager,
},
},
},
},
wantErr: false,
},
{
name: "rb with tolerations",
rb: &workv1alpha2.ResourceBinding{
TypeMeta: metav1.TypeMeta{
Kind: "ResourceBinding",
APIVersion: "work.karmada.io/v1alpha2",
},
ObjectMeta: metav1.ObjectMeta{
Name: "test-rb",
Namespace: "default",
Annotations: map[string]string{"policy.karmada.io/applied-placement": `{"clusterAffinity":{"clusterNames":["member1","member2"]},"clusterTolerations":[{"key":"cluster.karmada.io/not-ready","operator":"Exists","effect":"NoExecute","tolerationSeconds":30},{"key":"cluster.karmada.io/unreachable","operator":"Exists","effect":"NoExecute","tolerationSeconds":30}],"replicaScheduling":{"replicaSchedulingType":"Divided","replicaDivisionPreference":"Weighted","weightPreference":{"staticWeightList":[{"targetCluster":{"clusterNames":["member1"]},"weight":1},{"targetCluster":{"clusterNames":["member2"]},"weight":1}]}}}`},
},
Spec: workv1alpha2.ResourceBindingSpec{
Clusters: []workv1alpha2.TargetCluster{
{
Name: "test-cluster",
Replicas: 1,
},
},
},
},
cluster: &clusterv1alpha1.Cluster{
ObjectMeta: metav1.ObjectMeta{
Name: "test-cluster",
},
Spec: clusterv1alpha1.ClusterSpec{
Taints: []corev1.Taint{
{
Key: "cluster.karmada.io/not-ready",
Effect: corev1.TaintEffectNoExecute,
},
},
},
},
wrb: &workv1alpha2.ResourceBinding{
TypeMeta: metav1.TypeMeta{
Kind: "ResourceBinding",
APIVersion: "work.karmada.io/v1alpha2",
},
ObjectMeta: metav1.ObjectMeta{
Name: "test-rb",
Namespace: "default",
Annotations: map[string]string{"policy.karmada.io/applied-placement": `{"clusterAffinity":{"clusterNames":["member1","member2"]},"clusterTolerations":[{"key":"cluster.karmada.io/unreachable","operator":"Exists","effect":"NoExecute","tolerationSeconds":30}],"replicaScheduling":{"replicaSchedulingType":"Divided","replicaDivisionPreference":"Weighted","weightPreference":{"staticWeightList":[{"targetCluster":{"clusterNames":["member1"]},"weight":1},{"targetCluster":{"clusterNames":["member2"]},"weight":1}]}}}`},
},
Spec: workv1alpha2.ResourceBindingSpec{
Clusters: []workv1alpha2.TargetCluster{
{
Name: "test-cluster",
Replicas: 1,
},
},
},
},
wantErr: false,
},
{
name: "rb not exist",
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
tc := newNoExecuteTaintManager()
if tt.rb != nil {
if err := tc.Create(context.Background(), tt.rb, &client.CreateOptions{}); err != nil {
t.Fatalf("failed to create rb: %v", err)
}
}
if tt.cluster != nil {
if err := tc.Create(context.Background(), tt.cluster, &client.CreateOptions{}); err != nil {
t.Fatalf("failed to create cluster: %v", err)
}
}
key := keys.FederatedKey{
Cluster: "test-cluster",
ClusterWideKey: keys.ClusterWideKey{
Kind: "ResourceBinding",
Name: "test-rb",
Namespace: "default",
},
}
if err := tc.syncBindingEviction(key); (err != nil) != tt.wantErr {
t.Errorf("NoExecuteTaintManager.syncBindingEviction() error = %v, wantErr %v", err, tt.wantErr)
return
}
if tt.wrb != nil {
gRb := &workv1alpha2.ResourceBinding{}
if err := tc.Get(context.Background(), types.NamespacedName{Name: tt.wrb.Name, Namespace: tt.wrb.Namespace}, gRb, &client.GetOptions{}); err != nil {
t.Fatalf("failed to get rb, error %v", err)
}
if !reflect.DeepEqual(tt.wrb.Spec, gRb.Spec) {
t.Errorf("ResourceBinding get %+v, want %+v", gRb.Spec, tt.wrb.Spec)
}
}
})
}
}
func TestNoExecuteTaintManager_syncClusterBindingEviction(t *testing.T) {
replica := int32(1)
tests := []struct {
name string
crb *workv1alpha2.ClusterResourceBinding
cluster *clusterv1alpha1.Cluster
wcrb *workv1alpha2.ClusterResourceBinding
wantErr bool
}{
{
name: "crb without tolerations",
crb: &workv1alpha2.ClusterResourceBinding{
TypeMeta: metav1.TypeMeta{
Kind: "ClusterResourceBinding",
APIVersion: "work.karmada.io/v1alpha2",
},
ObjectMeta: metav1.ObjectMeta{
Name: "test-crb",
Annotations: map[string]string{"policy.karmada.io/applied-placement": `{"clusterAffinity":{"clusterNames":["member1","member2"]},"clusterTolerations":[{"key":"cluster.karmada.io/unreachable","operator":"Exists","effect":"NoExecute","tolerationSeconds":30}],"replicaScheduling":{"replicaSchedulingType":"Divided","replicaDivisionPreference":"Weighted","weightPreference":{"staticWeightList":[{"targetCluster":{"clusterNames":["member1"]},"weight":1},{"targetCluster":{"clusterNames":["member2"]},"weight":1}]}}}`},
},
Spec: workv1alpha2.ResourceBindingSpec{
Clusters: []workv1alpha2.TargetCluster{
{
Name: "test-cluster",
Replicas: 1,
},
},
},
},
cluster: &clusterv1alpha1.Cluster{
ObjectMeta: metav1.ObjectMeta{
Name: "test-cluster",
},
Spec: clusterv1alpha1.ClusterSpec{
Taints: []corev1.Taint{
{
Key: "cluster.karmada.io/not-ready",
Effect: corev1.TaintEffectNoExecute,
},
},
},
},
wcrb: &workv1alpha2.ClusterResourceBinding{
TypeMeta: metav1.TypeMeta{
Kind: "ClusterResourceBinding",
APIVersion: "work.karmada.io/v1alpha2",
},
ObjectMeta: metav1.ObjectMeta{
Name: "test-crb",
Annotations: map[string]string{"policy.karmada.io/applied-placement": `{"clusterAffinity":{"clusterNames":["member1","member2"]},"clusterTolerations":[{"key":"cluster.karmada.io/unreachable","operator":"Exists","effect":"NoExecute","tolerationSeconds":30}],"replicaScheduling":{"replicaSchedulingType":"Divided","replicaDivisionPreference":"Weighted","weightPreference":{"staticWeightList":[{"targetCluster":{"clusterNames":["member1"]},"weight":1},{"targetCluster":{"clusterNames":["member2"]},"weight":1}]}}}`},
},
Spec: workv1alpha2.ResourceBindingSpec{
GracefulEvictionTasks: []workv1alpha2.GracefulEvictionTask{
{
FromCluster: "test-cluster",
PurgeMode: policyv1alpha1.Graciously,
Replicas: &replica,
Reason: workv1alpha2.EvictionReasonTaintUntolerated,
Producer: workv1alpha2.EvictionProducerTaintManager,
},
},
},
},
wantErr: false,
},
{
name: "crb with tolerations",
crb: &workv1alpha2.ClusterResourceBinding{
TypeMeta: metav1.TypeMeta{
Kind: "ClusterResourceBinding",
APIVersion: "work.karmada.io/v1alpha2",
},
ObjectMeta: metav1.ObjectMeta{
Name: "test-crb",
Annotations: map[string]string{"policy.karmada.io/applied-placement": `{"clusterAffinity":{"clusterNames":["member1","member2"]},"clusterTolerations":[{"key":"cluster.karmada.io/not-ready","operator":"Exists","effect":"NoExecute","tolerationSeconds":30},{"key":"cluster.karmada.io/unreachable","operator":"Exists","effect":"NoExecute","tolerationSeconds":30}],"replicaScheduling":{"replicaSchedulingType":"Divided","replicaDivisionPreference":"Weighted","weightPreference":{"staticWeightList":[{"targetCluster":{"clusterNames":["member1"]},"weight":1},{"targetCluster":{"clusterNames":["member2"]},"weight":1}]}}}`},
},
Spec: workv1alpha2.ResourceBindingSpec{
Clusters: []workv1alpha2.TargetCluster{
{
Name: "test-cluster",
Replicas: 1,
},
},
},
},
cluster: &clusterv1alpha1.Cluster{
ObjectMeta: metav1.ObjectMeta{
Name: "test-cluster",
},
Spec: clusterv1alpha1.ClusterSpec{
Taints: []corev1.Taint{
{
Key: "cluster.karmada.io/not-ready",
Effect: corev1.TaintEffectNoExecute,
},
},
},
},
wcrb: &workv1alpha2.ClusterResourceBinding{
TypeMeta: metav1.TypeMeta{
Kind: "ClusterResourceBinding",
APIVersion: "work.karmada.io/v1alpha2",
},
ObjectMeta: metav1.ObjectMeta{
Name: "test-crb",
Annotations: map[string]string{"policy.karmada.io/applied-placement": `{"clusterAffinity":{"clusterNames":["member1","member2"]},"clusterTolerations":[{"key":"cluster.karmada.io/unreachable","operator":"Exists","effect":"NoExecute","tolerationSeconds":30}],"replicaScheduling":{"replicaSchedulingType":"Divided","replicaDivisionPreference":"Weighted","weightPreference":{"staticWeightList":[{"targetCluster":{"clusterNames":["member1"]},"weight":1},{"targetCluster":{"clusterNames":["member2"]},"weight":1}]}}}`},
},
Spec: workv1alpha2.ResourceBindingSpec{
Clusters: []workv1alpha2.TargetCluster{
{
Name: "test-cluster",
Replicas: 1,
},
},
},
},
wantErr: false,
},
{
name: "crb not exist",
wantErr: false,
},
{
name: "cluster not exist",
crb: &workv1alpha2.ClusterResourceBinding{
TypeMeta: metav1.TypeMeta{
Kind: "ClusterResourceBinding",
APIVersion: "work.karmada.io/v1alpha2",
},
ObjectMeta: metav1.ObjectMeta{
Name: "test-crb",
Annotations: map[string]string{"policy.karmada.io/applied-placement": `{"clusterAffinity":{"clusterNames":["member1","member2"]},"clusterTolerations":[{"key":"cluster.karmada.io/not-ready","operator":"Exists","effect":"NoExecute","tolerationSeconds":30},{"key":"cluster.karmada.io/unreachable","operator":"Exists","effect":"NoExecute","tolerationSeconds":30}],"replicaScheduling":{"replicaSchedulingType":"Divided","replicaDivisionPreference":"Weighted","weightPreference":{"staticWeightList":[{"targetCluster":{"clusterNames":["member1"]},"weight":1},{"targetCluster":{"clusterNames":["member2"]},"weight":1}]}}}`},
},
Spec: workv1alpha2.ResourceBindingSpec{
Clusters: []workv1alpha2.TargetCluster{
{
Name: "test-cluster",
Replicas: 1,
},
},
},
},
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
tc := newNoExecuteTaintManager()
if tt.crb != nil {
if err := tc.Create(context.Background(), tt.crb, &client.CreateOptions{}); err != nil {
t.Fatalf("failed to create crb: %v", err)
}
}
if tt.cluster != nil {
if err := tc.Create(context.Background(), tt.cluster, &client.CreateOptions{}); err != nil {
t.Fatalf("failed to create cluster: %v", err)
}
}
key := keys.FederatedKey{
Cluster: "test-cluster",
ClusterWideKey: keys.ClusterWideKey{
Kind: "ClusterResourceBinding",
Name: "test-crb",
},
}
if err := tc.syncClusterBindingEviction(key); (err != nil) != tt.wantErr {
t.Errorf("NoExecuteTaintManager.syncClusterBindingEviction() error = %v, wantErr %v", err, tt.wantErr)
}
if tt.wcrb != nil {
gCRB := &workv1alpha2.ClusterResourceBinding{}
if err := tc.Get(context.Background(), types.NamespacedName{Name: tt.wcrb.Name}, gCRB, &client.GetOptions{}); err != nil {
t.Fatalf("failed to get rb, error %v", err)
}
if !reflect.DeepEqual(tt.wcrb.Spec, gCRB.Spec) {
t.Errorf("ResourceBinding get %+v, want %+v", gCRB.Spec, tt.wcrb.Spec)
}
}
})
}
}