openkruise
/
kruise-game
mirror of https://github.com/openkruise/kruise-game.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fix: avoid patching gameserver continuously (#124) 

Signed-off-by: ChrisLiu <chrisliu1995@163.com>
This commit is contained in:
ChrisLiu 2024-01-18 17:00:31 +08:00 committed by GitHub
parent 02c00091d2
commit 2b6ce6fcfc
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 500 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
pkg/controllers/controller.go
View File

@ -17,10 +17,13 @@ limitations under the License.
package controller
import (
"context"
"github.com/openkruise/kruise-game/pkg/controllers/gameserver"
"github.com/openkruise/kruise-game/pkg/controllers/gameserverset"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/meta"
"k8s.io/klog/v2"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/manager"
)
@ -32,6 +35,13 @@ func init() {
}
func SetupWithManager(m manager.Manager) error {
if err := m.GetFieldIndexer().IndexField(context.Background(), &corev1.Pod{}, "spec.nodeName", func(rawObj client.Object) []string {
pod := rawObj.(*corev1.Pod)
return []string{pod.Spec.NodeName}
}); err != nil {
return err
}
for _, f := range controllerAddFuncs {
if err := f(m); err != nil {
if kindMatchErr, ok := err.(*meta.NoKindMatchError); ok {

85
pkg/controllers/gameserver/gameserver_conditions.go
View File

@ -172,18 +172,16 @@ func getPodConditions(pod *corev1.Pod) gamekruiseiov1alpha1.GameServerCondition
if message == "" && reason == "" {
return gamekruiseiov1alpha1.GameServerCondition{
Type: gamekruiseiov1alpha1.PodNormal,
Status: corev1.ConditionTrue,
LastProbeTime: metav1.Now(),
Type: gamekruiseiov1alpha1.PodNormal,
Status: corev1.ConditionTrue,
}
}
return gamekruiseiov1alpha1.GameServerCondition{
Type: gamekruiseiov1alpha1.PodNormal,
Status: corev1.ConditionFalse,
Reason: reason,
Message: message,
LastProbeTime: metav1.Now(),
Type: gamekruiseiov1alpha1.PodNormal,
Status: corev1.ConditionFalse,
Reason: reason,
Message: message,
}
}
@ -216,7 +214,7 @@ func getNodeConditions(node *corev1.Node) gamekruiseiov1alpha1.GameServerConditi
for _, condition := range node.Status.Conditions {
switch condition.Type {
case corev1.NodeReady:
case corev1.NodeReady, "SufficientIP":
if condition.Status != corev1.ConditionTrue {
message, reason = polyMessageReason(message, reason, condition.Message, string(condition.Type)+":"+condition.Reason)
}
@ -229,18 +227,16 @@ func getNodeConditions(node *corev1.Node) gamekruiseiov1alpha1.GameServerConditi
if message == "" && reason == "" {
return gamekruiseiov1alpha1.GameServerCondition{
Type: gamekruiseiov1alpha1.NodeNormal,
Status: corev1.ConditionTrue,
LastProbeTime: metav1.Now(),
Type: gamekruiseiov1alpha1.NodeNormal,
Status: corev1.ConditionTrue,
}
}
return gamekruiseiov1alpha1.GameServerCondition{
Type: gamekruiseiov1alpha1.NodeNormal,
Status: corev1.ConditionFalse,
Reason: reason,
Message: message,
LastProbeTime: metav1.Now(),
Type: gamekruiseiov1alpha1.NodeNormal,
Status: corev1.ConditionFalse,
Reason: reason,
Message: message,
}
}
@ -256,38 +252,34 @@ func getPersistentVolumeConditions(pvs []*corev1.PersistentVolume) gamekruiseiov
if message == "" && reason == "" {
return gamekruiseiov1alpha1.GameServerCondition{
Type: gamekruiseiov1alpha1.PersistentVolumeNormal,
Status: corev1.ConditionTrue,
LastProbeTime: metav1.Now(),
Type: gamekruiseiov1alpha1.PersistentVolumeNormal,
Status: corev1.ConditionTrue,
}
}
return gamekruiseiov1alpha1.GameServerCondition{
Type: gamekruiseiov1alpha1.PersistentVolumeNormal,
Status: corev1.ConditionFalse,
Reason: reason,
Message: message,
LastProbeTime: metav1.Now(),
Type: gamekruiseiov1alpha1.PersistentVolumeNormal,
Status: corev1.ConditionFalse,
Reason: reason,
Message: message,
}
}
func pvcNotFoundCondition(namespace, pvcName string) gamekruiseiov1alpha1.GameServerCondition {
return gamekruiseiov1alpha1.GameServerCondition{
Type: gamekruiseiov1alpha1.PersistentVolumeNormal,
Status: corev1.ConditionFalse,
Reason: pvcNotFoundReason,
Message: fmt.Sprintf("There is no pvc named %s/%s in cluster", namespace, pvcName),
LastProbeTime: metav1.Now(),
Type: gamekruiseiov1alpha1.PersistentVolumeNormal,
Status: corev1.ConditionFalse,
Reason: pvcNotFoundReason,
Message: fmt.Sprintf("There is no pvc named %s/%s in cluster", namespace, pvcName),
}
}
func pvNotFoundCondition(namespace, pvcName string) gamekruiseiov1alpha1.GameServerCondition {
return gamekruiseiov1alpha1.GameServerCondition{
Type: gamekruiseiov1alpha1.PersistentVolumeNormal,
Status: corev1.ConditionFalse,
Reason: pvNotFoundReason,
Message: fmt.Sprintf("There is no pv which pvc %s/%s is bound with", namespace, pvcName),
LastProbeTime: metav1.Now(),
Type: gamekruiseiov1alpha1.PersistentVolumeNormal,
Status: corev1.ConditionFalse,
Reason: pvNotFoundReason,
Message: fmt.Sprintf("There is no pv which pvc %s/%s is bound with", namespace, pvcName),
}
}
@ -371,3 +363,26 @@ func isConditionEqual(a, b gamekruiseiov1alpha1.GameServerCondition) bool {
}
return true
}
func isConditionsEqual(a, b []gamekruiseiov1alpha1.GameServerCondition) bool {
if len(a) != len(b) {
return false
}
for _, aCondition := range a {
found := false
for _, bCondition := range b {
if aCondition.Type == bCondition.Type {
found = true
if !isConditionEqual(aCondition, bCondition) {
return false
}
}
}
if !found {
return false
}
}
return true
}

42
pkg/controllers/gameserver/gameserver_controller.go
View File

@ -24,12 +24,16 @@ import (
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/fields"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/selection"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/tools/record"
"k8s.io/client-go/util/workqueue"
"k8s.io/klog/v2"
"k8s.io/utils/pointer"
"reflect"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller"
@ -80,6 +84,10 @@ func add(mgr manager.Manager, r reconcile.Reconciler) error {
klog.Error(err)
return err
}
if err = watchNode(c, mgr.GetClient()); err != nil {
klog.Error(err)
return err
}
return nil
}
@ -128,6 +136,40 @@ func watchPod(c controller.Controller) error {
return nil
}
func watchNode(c controller.Controller, cli client.Client) error {
if err := c.Watch(&source.Kind{Type: &corev1.Node{}}, &handler.Funcs{
UpdateFunc: func(updateEvent event.UpdateEvent, limitingInterface workqueue.RateLimitingInterface) {
nodeNew := updateEvent.ObjectNew.(*corev1.Node)
nodeOld := updateEvent.ObjectOld.(*corev1.Node)
if reflect.DeepEqual(nodeNew.Status.Conditions, nodeOld.Status.Conditions) {
return
}
podList := &corev1.PodList{}
ownerGss, _ := labels.NewRequirement(gamekruiseiov1alpha1.GameServerOwnerGssKey, selection.Exists, []string{})
err := cli.List(context.Background(), podList, &client.ListOptions{
LabelSelector: labels.NewSelector().Add(*ownerGss),
FieldSelector: fields.Set{"spec.nodeName": nodeNew.Name}.AsSelector(),
})
if err != nil {
klog.Errorf("List Pods By NodeName failed: %s", err.Error())
return
}
for _, pod := range podList.Items {
klog.Infof("Watch Node %s Conditions Changed, adding pods %s/%s in reconcile queue", nodeNew.Name, pod.Namespace, pod.Name)
limitingInterface.Add(reconcile.Request{
NamespacedName: types.NamespacedName{
Namespace: pod.GetNamespace(),
Name: pod.GetName(),
},
})
}
},
}); err != nil {
return err
}
return nil
}
//+kubebuilder:rbac:groups=game.kruise.io,resources=gameservers,verbs=get;list;watch;create;update;patch;delete
//+kubebuilder:rbac:groups=game.kruise.io,resources=gameservers/status,verbs=get;update;patch
//+kubebuilder:rbac:groups=game.kruise.io,resources=gameservers/finalizers,verbs=update

216
pkg/controllers/gameserver/gameserver_controller_test.go Normal file
View File

@ -0,0 +1,216 @@
/*
Copyright 2024 The Kruise Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package gameserver
import (
"context"
gameKruiseV1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1"
"github.com/openkruise/kruise-game/pkg/util"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/utils/pointer"
"reflect"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
"testing"
)
func TestGameServerReconcile(t *testing.T) {
nodeTemplate := &corev1.Node{
TypeMeta: metav1.TypeMeta{
Kind: "Node",
APIVersion: "v1",
},
}
gssTemplate := &gameKruiseV1alpha1.GameServerSet{
TypeMeta: metav1.TypeMeta{
Kind: "GameServerSet",
APIVersion: "game.kruise.io/v1alpha1",
},
ObjectMeta: metav1.ObjectMeta{
Namespace: "xxx",
Name: "xxx",
UID: "xxx-gss",
},
}
podTemplate := &corev1.Pod{
TypeMeta: metav1.TypeMeta{
Kind: "Pod",
APIVersion: "v1",
},
ObjectMeta: metav1.ObjectMeta{
Namespace: "xxx",
Name: "xxx-0",
UID: "xxx-pod",
Labels: map[string]string{
gameKruiseV1alpha1.GameServerOwnerGssKey: "xxx",
},
},
}
gsTemplate := &gameKruiseV1alpha1.GameServer{
TypeMeta: metav1.TypeMeta{
Kind: "GameServer",
APIVersion: "game.kruise.io/v1alpha1",
},
ObjectMeta: metav1.ObjectMeta{
Namespace: "xxx",
Name: "xxx-0",
UID: "xxx-gs",
Labels: map[string]string{
gameKruiseV1alpha1.GameServerOwnerGssKey: "xxx",
},
},
}
tests := []struct {
req ctrl.Request
getGss func() *gameKruiseV1alpha1.GameServerSet
getPod func() *corev1.Pod
getGs func() *gameKruiseV1alpha1.GameServer
getNode func() *corev1.Node
getExpectGs func() *gameKruiseV1alpha1.GameServer
}{
{
req: ctrl.Request{
NamespacedName: types.NamespacedName{
Name: "xxx-0",
Namespace: "xxx",
},
},
getGss: func() *gameKruiseV1alpha1.GameServerSet {
return gssTemplate.DeepCopy()
},
getPod: func() *corev1.Pod {
return podTemplate.DeepCopy()
},
getGs: func() *gameKruiseV1alpha1.GameServer {
return nil
},
getNode: func() *corev1.Node {
return nodeTemplate.DeepCopy()
},
getExpectGs: func() *gameKruiseV1alpha1.GameServer {
gs := gsTemplate.DeepCopy()
gs.Annotations = make(map[string]string)
gs.Annotations[gameKruiseV1alpha1.GsTemplateMetadataHashKey] = util.GetGsTemplateMetadataHash(gssTemplate)
gs.OwnerReferences = []metav1.OwnerReference{
{
APIVersion: podTemplate.APIVersion,
Kind: podTemplate.Kind,
Name: podTemplate.GetName(),
UID: podTemplate.GetUID(),
Controller: pointer.BoolPtr(true),
BlockOwnerDeletion: pointer.BoolPtr(true),
},
}
updatePriority := intstr.FromInt(0)
deletionPriority := intstr.FromInt(0)
gs.Spec = gameKruiseV1alpha1.GameServerSpec{
DeletionPriority: &deletionPriority,
UpdatePriority: &updatePriority,
OpsState: gameKruiseV1alpha1.None,
NetworkDisabled: false,
}
return gs
},
},
{
req: ctrl.Request{
NamespacedName: types.NamespacedName{
Name: "xxx-0",
Namespace: "xxx",
},
},
getGss: func() *gameKruiseV1alpha1.GameServerSet {
return gssTemplate.DeepCopy()
},
getPod: func() *corev1.Pod {
return nil
},
getGs: func() *gameKruiseV1alpha1.GameServer {
gs := gsTemplate.DeepCopy()
gs.GetLabels()[gameKruiseV1alpha1.GameServerDeletingKey] = "true"
return gs
},
getNode: func() *corev1.Node {
return nodeTemplate.DeepCopy()
},
getExpectGs: func() *gameKruiseV1alpha1.GameServer {
return nil
},
},
}
for i, test := range tests {
objs := []client.Object{test.getNode(), test.getGss()}
pod := test.getPod()
gs := test.getGs()
if pod != nil {
objs = append(objs, pod)
}
if gs != nil {
objs = append(objs, gs)
}
c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(objs...).Build()
recon := GameServerReconciler{Client: c}
if _, err := recon.Reconcile(context.TODO(), test.req); err != nil {
t.Error(err)
}
expectGs := test.getExpectGs()
actualGs := &gameKruiseV1alpha1.GameServer{}
if err := c.Get(context.TODO(), test.req.NamespacedName, actualGs); err != nil {
if expectGs == nil && errors.IsNotFound(err) {
continue
}
t.Error(err)
}
// gs labels
expectGsLabels := expectGs.GetLabels()
actualGsLabels := actualGs.GetLabels()
if !reflect.DeepEqual(expectGsLabels, actualGsLabels) {
t.Errorf("case %d: expect labels %v, but actually got %v", i, expectGsLabels, actualGsLabels)
}
// gs annotations
expectGsAnnotations := expectGs.GetAnnotations()
actualGsAnnotations := actualGs.GetAnnotations()
if !reflect.DeepEqual(expectGsAnnotations, actualGsAnnotations) {
t.Errorf("case %d: expect annotations %v, but actually got %v", i, expectGsAnnotations, actualGsAnnotations)
}
// gs ownerReferences
expectGsOwnerReferences := expectGs.GetOwnerReferences()
actualGsOwnerReferences := actualGs.GetOwnerReferences()
if !reflect.DeepEqual(expectGsOwnerReferences, actualGsOwnerReferences) {
t.Errorf("case %d: expect ownerReferences %v, but actually got %v", i, expectGsOwnerReferences, actualGsOwnerReferences)
}
// gs spec
expectGsSpec := expectGs.Spec
actualGsSpec := actualGs.Spec
if !reflect.DeepEqual(expectGsSpec, actualGsSpec) {
t.Errorf("case %d: expect Spec %v, but actually got %v", i, expectGsSpec, actualGsSpec)
}
}
}

59
pkg/controllers/gameserver/gameserver_manager.go
View File

@ -210,24 +210,23 @@ func (manager GameServerManager) SyncPodToGs(gss *gameKruiseV1alpha1.GameServerS
podGsState := gameKruiseV1alpha1.GameServerState(podLabels[gameKruiseV1alpha1.GameServerStateKey])
// sync Service Qualities
spec, newGsConditions := syncServiceQualities(gss.Spec.ServiceQualities, pod.Status.Conditions, gs.Status.ServiceQualitiesCondition)
spec, sqConditions := syncServiceQualities(gss.Spec.ServiceQualities, pod.Status.Conditions, gs.Status.ServiceQualitiesCondition)
// sync metadata
var gsMetadata metav1.ObjectMeta
if isNeedToSyncMetadata(gss, gs) {
gsMetadata = syncMetadataFromGss(gss)
}
if isNeedToSyncMetadata(gss, gs) || !reflect.DeepEqual(spec, gs.Spec) {
// sync metadata
gsMetadata := syncMetadataFromGss(gss)
// patch gs spec
patchSpec := map[string]interface{}{"spec": spec, "metadata": gsMetadata}
jsonPatchSpec, err := json.Marshal(patchSpec)
if err != nil {
return err
}
err = manager.client.Patch(context.TODO(), gs, client.RawPatch(types.MergePatchType, jsonPatchSpec))
if err != nil && !errors.IsNotFound(err) {
klog.Errorf("failed to patch GameServer spec %s in %s,because of %s.", gs.GetName(), gs.GetNamespace(), err.Error())
return err
// patch gs spec & metadata
patchSpec := map[string]interface{}{"spec": spec, "metadata": gsMetadata}
jsonPatchSpec, err := json.Marshal(patchSpec)
if err != nil {
return err
}
err = manager.client.Patch(context.TODO(), gs, client.RawPatch(types.MergePatchType, jsonPatchSpec))
if err != nil && !errors.IsNotFound(err) {
klog.Errorf("failed to patch GameServer spec %s in %s,because of %s.", gs.GetName(), gs.GetNamespace(), err.Error())
return err
}
}
// get gs conditions
@ -238,26 +237,30 @@ func (manager GameServerManager) SyncPodToGs(gss *gameKruiseV1alpha1.GameServerS
}
// patch gs status
status := gameKruiseV1alpha1.GameServerStatus{
oldStatus := *gs.Status.DeepCopy()
newStatus := gameKruiseV1alpha1.GameServerStatus{
PodStatus: pod.Status,
CurrentState: podGsState,
DesiredState: gameKruiseV1alpha1.Ready,
UpdatePriority: &podUpdatePriority,
DeletionPriority: &podDeletePriority,
ServiceQualitiesCondition: newGsConditions,
ServiceQualitiesCondition: sqConditions,
NetworkStatus: manager.syncNetworkStatus(),
LastTransitionTime: metav1.Now(),
LastTransitionTime: oldStatus.LastTransitionTime,
Conditions: conditions,
}
patchStatus := map[string]interface{}{"status": status}
jsonPatchStatus, err := json.Marshal(patchStatus)
if err != nil {
return err
}
err = manager.client.Status().Patch(context.TODO(), gs, client.RawPatch(types.MergePatchType, jsonPatchStatus))
if err != nil && !errors.IsNotFound(err) {
klog.Errorf("failed to patch GameServer Status %s in %s,because of %s.", gs.GetName(), gs.GetNamespace(), err.Error())
return err
if !reflect.DeepEqual(oldStatus, newStatus) {
newStatus.LastTransitionTime = metav1.Now()
patchStatus := map[string]interface{}{"status": newStatus}
jsonPatchStatus, err := json.Marshal(patchStatus)
if err != nil {
return err
}
err = manager.client.Status().Patch(context.TODO(), gs, client.RawPatch(types.MergePatchType, jsonPatchStatus))
if err != nil && !errors.IsNotFound(err) {
klog.Errorf("failed to patch GameServer Status %s in %s,because of %s.", gs.GetName(), gs.GetNamespace(), err.Error())
return err
}
}
return nil

151
pkg/controllers/gameserver/gameserver_manager_test.go
View File

@ -743,3 +743,154 @@ func TestSyncPodContainers(t *testing.T) {
}
}
}
func TestSyncPodToGs(t *testing.T) {
tests := []struct {
gs *gameKruiseV1alpha1.GameServer
pod *corev1.Pod
gss *gameKruiseV1alpha1.GameServerSet
node *corev1.Node
gsStatus gameKruiseV1alpha1.GameServerStatus
}{
{
gss: &gameKruiseV1alpha1.GameServerSet{
ObjectMeta: metav1.ObjectMeta{
Namespace: "xxx",
Name: "xxx",
},
Spec: gameKruiseV1alpha1.GameServerSetSpec{
GameServerTemplate: gameKruiseV1alpha1.GameServerTemplate{
PodTemplateSpec: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Labels: map[string]string{
"key-0": "value-0",
},
},
},
},
},
},
gs: &gameKruiseV1alpha1.GameServer{
ObjectMeta: metav1.ObjectMeta{
Namespace: "xxx",
Name: "xxx-0",
Labels: map[string]string{
gameKruiseV1alpha1.GameServerOwnerGssKey: "xxx",
},
},
Status: gameKruiseV1alpha1.GameServerStatus{
CurrentState: gameKruiseV1alpha1.Creating,
},
},
pod: &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Namespace: "xxx",
Name: "xxx-0",
Labels: map[string]string{
gameKruiseV1alpha1.GameServerOpsStateKey: string(gameKruiseV1alpha1.WaitToDelete),
gameKruiseV1alpha1.GameServerStateKey: string(gameKruiseV1alpha1.Ready),
},
},
Spec: corev1.PodSpec{
NodeName: "node-A",
},
Status: corev1.PodStatus{
Conditions: []corev1.PodCondition{
{
Type: "Ready",
Status: "True",
},
{
Type: "PodScheduled",
Status: "True",
},
{
Type: "ContainersReady",
Status: "True",
},
},
},
},
node: &corev1.Node{
ObjectMeta: metav1.ObjectMeta{
Name: "node-A",
},
Status: corev1.NodeStatus{
Conditions: []corev1.NodeCondition{
{
Type: "Ready",
Status: "True",
},
{
Type: "PIDPressure",
Status: "False",
},
{
Type: "SufficientIP",
Status: "True",
},
{
Type: "RuntimeOffline",
Status: "False",
},
{
Type: "DockerOffline",
Status: "False",
},
},
},
},
gsStatus: gameKruiseV1alpha1.GameServerStatus{
Conditions: []gameKruiseV1alpha1.GameServerCondition{
{
Type: "PodNormal",
Status: "True",
},
{
Type: "NodeNormal",
Status: "True",
},
{
Type: "PersistentVolumeNormal",
Status: "True",
},
},
},
},
}
for i, test := range tests {
objs := []client.Object{test.gs, test.pod, test.node, test.gss}
c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(objs...).Build()
manager := &GameServerManager{
client: c,
gameServer: test.gs,
pod: test.pod,
}
if err := manager.SyncPodToGs(test.gss); err != nil {
t.Error(err)
}
gs := &gameKruiseV1alpha1.GameServer{}
if err := manager.client.Get(context.TODO(), types.NamespacedName{
Namespace: test.gs.Namespace,
Name: test.gs.Name,
}, gs); err != nil {
t.Error(err)
}
// gs metadata
gsLabels := gs.GetLabels()
for key, value := range test.gss.Spec.GameServerTemplate.GetLabels() {
if gsLabels[key] != value {
t.Errorf("case %d: expect label %s=%s exists on gs, but actually not", i, key, value)
}
}
// gs status conditions
if !isConditionsEqual(test.gsStatus.Conditions, gs.Status.Conditions) {
t.Errorf("case %d: expect conditions is %v, but actually %v", i, test.gsStatus.Conditions, gs.Status.Conditions)
}
}
}