karmada/pkg/search/proxy/store/multi_cluster_cache_test.go

1458 lines
39 KiB
Go

package store
import (
"context"
"fmt"
"reflect"
"strings"
"sync"
"testing"
"time"
"github.com/google/go-cmp/cmp"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/meta"
metainternalversion "k8s.io/apimachinery/pkg/apis/meta/internalversion"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apiserver/pkg/endpoints/request"
"k8s.io/client-go/dynamic"
fakedynamic "k8s.io/client-go/dynamic/fake"
kubetesting "k8s.io/client-go/testing"
clusterv1alpha1 "github.com/karmada-io/karmada/pkg/apis/cluster/v1alpha1"
)
var (
podGVR = corev1.SchemeGroupVersion.WithResource("pods")
nodeGVR = corev1.SchemeGroupVersion.WithResource("nodes")
secretGVR = corev1.SchemeGroupVersion.WithResource("secrets")
podGVK = corev1.SchemeGroupVersion.WithKind("Pod")
nodeGVK = corev1.SchemeGroupVersion.WithKind("Node")
secretGVK = corev1.SchemeGroupVersion.WithKind("Secret")
restMapper = meta.NewDefaultRESTMapper([]schema.GroupVersion{corev1.SchemeGroupVersion})
scheme = runtime.NewScheme()
)
func init() {
restMapper.Add(podGVK, meta.RESTScopeNamespace)
restMapper.Add(nodeGVK, meta.RESTScopeRoot)
restMapper.Add(secretGVK, meta.RESTScopeNamespace)
scheme.AddKnownTypes(corev1.SchemeGroupVersion,
&corev1.Pod{}, &corev1.PodList{},
&corev1.Node{}, &corev1.NodeList{},
&corev1.Secret{}, &corev1.SecretList{})
}
func TestMultiClusterCache_UpdateCache(t *testing.T) {
newClientFunc := func(cluster string) (dynamic.Interface, error) {
return fakedynamic.NewSimpleDynamicClient(scheme), nil
}
cache := NewMultiClusterCache(newClientFunc, restMapper)
defer cache.Stop()
cluster1 := newCluster("cluster1")
cluster2 := newCluster("cluster2")
resources := map[string]map[schema.GroupVersionResource]*MultiNamespace{
cluster1.Name: resourceSet(podGVR, nodeGVR),
cluster2.Name: resourceSet(podGVR),
}
err := cache.UpdateCache(resources)
if err != nil {
t.Error(err)
}
if len(cache.cache) != 2 {
t.Errorf("cache len expect %v, actual %v", 2, len(cache.cache))
}
// Then test removing cluster2 and remove node cache for cluster1
err = cache.UpdateCache(map[string]map[schema.GroupVersionResource]*MultiNamespace{
cluster1.Name: resourceSet(podGVR),
})
if err != nil {
t.Error(err)
}
if len(cache.cache) != 1 {
t.Errorf("cache len expect %v, actual %v", 1, len(cache.cache))
}
}
func TestMultiClusterCache_HasResource(t *testing.T) {
fakeClient := fakedynamic.NewSimpleDynamicClient(scheme)
newClientFunc := func(cluster string) (dynamic.Interface, error) {
return fakeClient, nil
}
cache := NewMultiClusterCache(newClientFunc, restMapper)
defer cache.Stop()
cluster1 := newCluster("cluster1")
cluster2 := newCluster("cluster2")
resources := map[string]map[schema.GroupVersionResource]*MultiNamespace{
cluster1.Name: resourceSet(podGVR, nodeGVR),
cluster2.Name: resourceSet(podGVR),
}
err := cache.UpdateCache(resources)
if err != nil {
t.Error(err)
return
}
tests := []struct {
name string
args schema.GroupVersionResource
want bool
}{
{
"has gets",
podGVR,
true,
},
{
"has nodes",
nodeGVR,
true,
},
{
"has no secret",
secretGVR,
false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
actual := cache.HasResource(tt.args)
if tt.want != actual {
t.Errorf("should return %v", tt.want)
}
})
}
}
func TestMultiClusterCache_GetResourceFromCache(t *testing.T) {
cluster1 := newCluster("cluster1")
cluster2 := newCluster("cluster2")
resources := map[string]map[schema.GroupVersionResource]*MultiNamespace{
cluster1.Name: resourceSet(podGVR),
cluster2.Name: resourceSet(podGVR),
}
cluster1Client := fakedynamic.NewSimpleDynamicClient(scheme,
newUnstructuredObject(podGVK, "pod11", withDefaultNamespace()),
newUnstructuredObject(podGVK, "pod_conflict", withDefaultNamespace()),
)
cluster2Client := fakedynamic.NewSimpleDynamicClient(scheme,
newUnstructuredObject(podGVK, "pod21", withDefaultNamespace()),
newUnstructuredObject(podGVK, "pod_conflict", withDefaultNamespace()),
)
newClientFunc := func(cluster string) (dynamic.Interface, error) {
switch cluster {
case cluster1.Name:
return cluster1Client, nil
case cluster2.Name:
return cluster2Client, nil
}
return fakedynamic.NewSimpleDynamicClient(scheme), nil
}
cache := NewMultiClusterCache(newClientFunc, restMapper)
defer cache.Stop()
err := cache.UpdateCache(resources)
if err != nil {
t.Error(err)
return
}
type args struct {
gvr schema.GroupVersionResource
namespace string
name string
}
type want struct {
objectName string
cluster string
errAssert func(error) bool
}
tests := []struct {
name string
args args
want want
}{
{
name: "get pod from cluster1",
args: args{
gvr: podGVR,
namespace: metav1.NamespaceDefault,
name: "pod11",
},
want: want{
objectName: "pod11",
cluster: "cluster1",
errAssert: noError,
},
},
{
name: "get pod from cluster2",
args: args{
gvr: podGVR,
namespace: metav1.NamespaceDefault,
name: "pod21",
},
want: want{
objectName: "pod21",
cluster: "cluster2",
errAssert: noError,
},
},
{
name: "pod not found",
args: args{
gvr: podGVR,
namespace: metav1.NamespaceDefault,
name: "podz",
},
want: want{
errAssert: apierrors.IsNotFound,
},
},
{
name: "get resource conflict",
args: args{
gvr: podGVR,
namespace: metav1.NamespaceDefault,
name: "pod_conflict",
},
want: want{
errAssert: apierrors.IsConflict,
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
obj, cluster, err := cache.GetResourceFromCache(context.TODO(), tt.args.gvr, tt.args.namespace, tt.args.name)
if !tt.want.errAssert(err) {
t.Errorf("Unexpect error: %v", err)
return
}
if err != nil {
return
}
accessor, err := meta.Accessor(obj)
if err != nil {
t.Error(err)
return
}
if tt.want.objectName != accessor.GetName() {
t.Errorf("Expect object %v. But got %v", tt.want.objectName, accessor.GetName())
}
if !reflect.DeepEqual(tt.want.cluster, cluster) {
t.Errorf("Cluster diff: %v", cmp.Diff(tt.want.cluster, cluster))
}
})
}
}
func TestMultiClusterCache_Get(t *testing.T) {
cluster1 := newCluster("cluster1")
cluster2 := newCluster("cluster2")
cluster1Client := fakedynamic.NewSimpleDynamicClient(scheme,
newUnstructuredObject(podGVK, "pod11", withDefaultNamespace(), withResourceVersion("1000")),
newUnstructuredObject(nodeGVK, "node11", withResourceVersion("1000")),
newUnstructuredObject(podGVK, "pod-conflict", withDefaultNamespace(), withResourceVersion("1000")),
)
cluster2Client := fakedynamic.NewSimpleDynamicClient(scheme,
newUnstructuredObject(podGVK, "pod21", withDefaultNamespace(), withResourceVersion("2000")),
newUnstructuredObject(podGVK, "pod-conflict", withDefaultNamespace(), withResourceVersion("2000")),
)
newClientFunc := func(cluster string) (dynamic.Interface, error) {
switch cluster {
case cluster1.Name:
return cluster1Client, nil
case cluster2.Name:
return cluster2Client, nil
}
return fakedynamic.NewSimpleDynamicClient(scheme), nil
}
cache := NewMultiClusterCache(newClientFunc, restMapper)
defer cache.Stop()
err := cache.UpdateCache(map[string]map[schema.GroupVersionResource]*MultiNamespace{
cluster1.Name: resourceSet(podGVR, nodeGVR),
cluster2.Name: resourceSet(podGVR),
})
if err != nil {
t.Fatal(err)
}
type args struct {
ctx context.Context
gvr schema.GroupVersionResource
name string
options *metav1.GetOptions
}
type want struct {
object runtime.Object
errAssert func(error) bool
}
tests := []struct {
name string
args args
want want
}{
{
name: "get pod11 from cluster1",
args: args{
ctx: request.WithNamespace(context.TODO(), metav1.NamespaceDefault),
gvr: podGVR,
name: "pod11",
options: &metav1.GetOptions{},
},
want: want{
object: newUnstructuredObject(podGVK, "pod11", withDefaultNamespace(), withResourceVersion(buildMultiClusterRV("cluster1", "1000")), withCacheSourceAnnotation("cluster1")),
errAssert: noError,
},
},
{
name: "get pod21 from cluster2",
args: args{
ctx: request.WithNamespace(context.TODO(), metav1.NamespaceDefault),
gvr: podGVR,
name: "pod21",
options: &metav1.GetOptions{},
},
want: want{
object: newUnstructuredObject(podGVK, "pod21", withDefaultNamespace(), withResourceVersion(buildMultiClusterRV("cluster2", "2000")), withCacheSourceAnnotation("cluster2")),
errAssert: noError,
},
},
{
name: "get pod not found",
args: args{
ctx: request.WithNamespace(context.TODO(), metav1.NamespaceDefault),
gvr: podGVR,
name: "podz",
options: &metav1.GetOptions{},
},
want: want{
object: nil,
errAssert: apierrors.IsNotFound,
},
},
{
name: "get pod with large resource version",
args: args{
ctx: request.WithNamespace(context.TODO(), metav1.NamespaceDefault),
gvr: podGVR,
name: "pod11",
options: &metav1.GetOptions{ResourceVersion: "9999"},
},
want: want{
object: nil,
errAssert: apierrors.IsTimeout,
},
},
{
name: "get node from cluster1",
args: args{
ctx: context.TODO(),
gvr: nodeGVR,
name: "node11",
options: &metav1.GetOptions{},
},
want: want{
object: newUnstructuredObject(nodeGVK, "node11", withResourceVersion(buildMultiClusterRV("cluster1", "1000")), withCacheSourceAnnotation("cluster1")),
errAssert: noError,
},
},
{
name: "get pod conflict",
args: args{
ctx: request.WithNamespace(context.TODO(), metav1.NamespaceDefault),
gvr: podGVR,
name: "pod-conflict",
options: &metav1.GetOptions{},
},
want: want{
errAssert: apierrors.IsConflict,
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
obj, err := cache.Get(tt.args.ctx, tt.args.gvr, tt.args.name, tt.args.options)
if !tt.want.errAssert(err) {
t.Errorf("Unexpected error: %v", err)
return
}
if err != nil {
return
}
if !reflect.DeepEqual(tt.want.object, obj) {
t.Errorf("Objects diff: %v", cmp.Diff(tt.want.object, obj))
}
})
}
}
func TestMultiClusterCache_Get_Namespaced(t *testing.T) {
cluster1 := newCluster("cluster1")
cluster2 := newCluster("cluster2")
cluster1Client := fakedynamic.NewSimpleDynamicClient(scheme,
newUnstructuredObject(podGVK, "pod11", withNamespace("ns1"), withResourceVersion("1000")),
newUnstructuredObject(podGVK, "pod12", withNamespace("ns2"), withResourceVersion("1000")),
)
cluster2Client := fakedynamic.NewSimpleDynamicClient(scheme,
newUnstructuredObject(podGVK, "pod21", withNamespace("ns1"), withResourceVersion("2000")),
newUnstructuredObject(podGVK, "pod22", withNamespace("ns2"), withResourceVersion("2000")),
newUnstructuredObject(podGVK, "pod23", withNamespace("ns3"), withResourceVersion("2000")),
)
newClientFunc := func(cluster string) (dynamic.Interface, error) {
switch cluster {
case cluster1.Name:
return cluster1Client, nil
case cluster2.Name:
return cluster2Client, nil
}
return fakedynamic.NewSimpleDynamicClient(scheme), nil
}
cache := NewMultiClusterCache(newClientFunc, restMapper)
defer cache.Stop()
err := cache.UpdateCache(map[string]map[schema.GroupVersionResource]*MultiNamespace{
cluster1.Name: {
podGVR: &MultiNamespace{namespaces: sets.New[string]("ns1")},
},
cluster2.Name: {
podGVR: &MultiNamespace{namespaces: sets.New[string]("ns1", "ns2")},
},
})
if err != nil {
t.Fatal(err)
}
type args struct {
ns string
gvr schema.GroupVersionResource
name string
options *metav1.GetOptions
}
type want struct {
object runtime.Object
errAssert func(error) bool
}
tests := []struct {
name string
args args
want want
}{
{
name: "pod is cached in single ns cache",
args: args{
ns: "ns1",
gvr: podGVR,
name: "pod11",
options: &metav1.GetOptions{},
},
want: want{
object: newUnstructuredObject(podGVK, "pod11", withNamespace("ns1"), withResourceVersion(buildMultiClusterRV("cluster1", "1000")), withCacheSourceAnnotation("cluster1")),
errAssert: noError,
},
},
{
name: "pod is not cached in single ns cache",
args: args{
ns: "ns1",
gvr: podGVR,
name: "pod12",
options: &metav1.GetOptions{},
},
want: want{
object: nil,
errAssert: apierrors.IsNotFound,
},
},
{
name: "pod is cached in multi ns cache",
args: args{
ns: "ns1",
gvr: podGVR,
name: "pod21",
options: &metav1.GetOptions{},
},
want: want{
object: newUnstructuredObject(podGVK, "pod21", withNamespace("ns1"), withResourceVersion(buildMultiClusterRV("cluster2", "2000")), withCacheSourceAnnotation("cluster2")),
errAssert: noError,
},
},
{
name: "pod not found in cache.",
args: args{
ns: "ns1",
gvr: podGVR,
name: "non-exist",
options: &metav1.GetOptions{},
},
want: want{
object: nil,
errAssert: apierrors.IsNotFound,
},
},
{
name: "ns is not cached",
args: args{
ns: "ns3",
gvr: podGVR,
name: "pod23",
options: &metav1.GetOptions{},
},
want: want{
object: nil,
errAssert: apierrors.IsNotFound,
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
obj, err := cache.Get(request.WithNamespace(context.TODO(), tt.args.ns), tt.args.gvr, tt.args.name, tt.args.options)
if !tt.want.errAssert(err) {
t.Errorf("Unexpected error: %v", err)
return
}
if err != nil {
return
}
if !reflect.DeepEqual(tt.want.object, obj) {
t.Errorf("Objects diff: %v", cmp.Diff(tt.want.object, obj))
}
})
}
}
func TestMultiClusterCache_List(t *testing.T) {
cluster1 := newCluster("cluster1")
cluster2 := newCluster("cluster2")
cluster1Client := fakedynamic.NewSimpleDynamicClient(scheme,
newUnstructuredObject(podGVK, "pod11", withDefaultNamespace(), withResourceVersion("1001"), withLabel("app", "foo")),
newUnstructuredObject(podGVK, "pod12", withDefaultNamespace(), withResourceVersion("1002")),
newUnstructuredObject(podGVK, "pod13", withDefaultNamespace(), withResourceVersion("1003"), withLabel("app", "foo")),
newUnstructuredObject(podGVK, "pod14", withDefaultNamespace(), withResourceVersion("1004")),
newUnstructuredObject(podGVK, "pod15", withDefaultNamespace(), withResourceVersion("1005")),
)
cluster2Client := fakedynamic.NewSimpleDynamicClient(scheme,
newUnstructuredObject(podGVK, "pod21", withDefaultNamespace(), withResourceVersion("2001"), withLabel("app", "foo")),
newUnstructuredObject(podGVK, "pod22", withDefaultNamespace(), withResourceVersion("2002")),
newUnstructuredObject(podGVK, "pod23", withDefaultNamespace(), withResourceVersion("2003"), withLabel("app", "foo")),
newUnstructuredObject(podGVK, "pod24", withDefaultNamespace(), withResourceVersion("2004")),
newUnstructuredObject(podGVK, "pod25", withDefaultNamespace(), withResourceVersion("2005")),
)
newClientFunc := func(cluster string) (dynamic.Interface, error) {
switch cluster {
case cluster1.Name:
return cluster1Client, nil
case cluster2.Name:
return cluster2Client, nil
}
return fakedynamic.NewSimpleDynamicClient(scheme), nil
}
type args struct {
ctx context.Context
gvr schema.GroupVersionResource
options *metainternalversion.ListOptions
}
type want struct {
resourceVersion string
names sets.Set[string]
errAssert func(error) bool
}
tests := []struct {
name string
resources map[string]map[schema.GroupVersionResource]*MultiNamespace
args args
want want
}{
{
name: "list gets with labelSelector",
resources: map[string]map[schema.GroupVersionResource]*MultiNamespace{},
args: args{
ctx: request.WithNamespace(context.TODO(), metav1.NamespaceDefault),
gvr: podGVR,
options: &metainternalversion.ListOptions{
LabelSelector: asLabelSelector("app=foo"),
},
},
want: want{
// fakeDynamic returns list with resourceVersion=""
resourceVersion: "",
names: sets.New[string](),
errAssert: noError,
},
},
{
name: "list gets",
resources: map[string]map[schema.GroupVersionResource]*MultiNamespace{
cluster1.Name: resourceSet(podGVR),
cluster2.Name: resourceSet(podGVR),
},
args: args{
ctx: request.WithNamespace(context.TODO(), metav1.NamespaceDefault),
gvr: podGVR,
options: &metainternalversion.ListOptions{},
},
want: want{
// fakeDynamic returns list with resourceVersion=""
resourceVersion: buildMultiClusterRV("cluster1", "", "cluster2", ""),
names: sets.New[string]("pod11", "pod12", "pod13", "pod14", "pod15", "pod21", "pod22", "pod23", "pod24", "pod25"),
errAssert: noError,
},
},
{
name: "list gets with labelSelector",
resources: map[string]map[schema.GroupVersionResource]*MultiNamespace{
cluster1.Name: resourceSet(podGVR),
cluster2.Name: resourceSet(podGVR),
},
args: args{
ctx: request.WithNamespace(context.TODO(), metav1.NamespaceDefault),
gvr: podGVR,
options: &metainternalversion.ListOptions{
LabelSelector: asLabelSelector("app=foo"),
},
},
want: want{
// fakeDynamic returns list with resourceVersion=""
resourceVersion: buildMultiClusterRV("cluster1", "", "cluster2", ""),
names: sets.New[string]("pod11", "pod13", "pod21", "pod23"),
errAssert: noError,
},
},
// TODO: add case for limit option. But fakeClient doesn't support it.
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
cache := NewMultiClusterCache(newClientFunc, restMapper)
defer cache.Stop()
err := cache.UpdateCache(tt.resources)
if err != nil {
t.Error(err)
return
}
obj, err := cache.List(tt.args.ctx, tt.args.gvr, tt.args.options)
if !tt.want.errAssert(err) {
t.Errorf("Unexpected error: %v", err)
return
}
if err != nil {
return
}
object, err := meta.ListAccessor(obj)
if err != nil {
t.Error(err)
return
}
if tt.want.resourceVersion != object.GetResourceVersion() {
t.Errorf("ResourceVersion want=%v, actual=%v", tt.want.resourceVersion, object.GetResourceVersion())
}
names := sets.New[string]()
err = meta.EachListItem(obj, func(o runtime.Object) error {
a, err := meta.Accessor(o)
if err != nil {
return err
}
names.Insert(a.GetName())
return nil
})
if err != nil {
t.Error(err)
return
}
if !tt.want.names.Equal(names) {
t.Errorf("List items want=%v, actual=%v", strings.Join(sets.List(tt.want.names), ","), strings.Join(sets.List(names), ","))
}
})
}
}
func TestMultiClusterCache_List_CacheSourceAnnotation(t *testing.T) {
cluster1 := newCluster("cluster1")
cluster2 := newCluster("cluster2")
cluster1Client := fakedynamic.NewSimpleDynamicClient(scheme,
newUnstructuredObject(podGVK, "pod11"),
newUnstructuredObject(podGVK, "pod12"),
)
cluster2Client := fakedynamic.NewSimpleDynamicClient(scheme,
newUnstructuredObject(podGVK, "pod21"),
newUnstructuredObject(podGVK, "pod22"),
)
newClientFunc := func(cluster string) (dynamic.Interface, error) {
switch cluster {
case cluster1.Name:
return cluster1Client, nil
case cluster2.Name:
return cluster2Client, nil
}
return fakedynamic.NewSimpleDynamicClient(scheme), nil
}
cache := NewMultiClusterCache(newClientFunc, restMapper)
defer cache.Stop()
err := cache.UpdateCache(map[string]map[schema.GroupVersionResource]*MultiNamespace{
cluster1.Name: resourceSet(podGVR),
cluster2.Name: resourceSet(podGVR),
})
if err != nil {
t.Error(err)
return
}
list, err := cache.List(context.TODO(), podGVR, &metainternalversion.ListOptions{})
if err != nil {
t.Errorf("List error: %v", err)
return
}
items, err := meta.ExtractList(list)
if err != nil {
t.Errorf("ExtractList error: %v", err)
return
}
expect := []runtime.Object{
newUnstructuredObject(podGVK, "pod11", withCacheSourceAnnotation("cluster1")),
newUnstructuredObject(podGVK, "pod12", withCacheSourceAnnotation("cluster1")),
newUnstructuredObject(podGVK, "pod21", withCacheSourceAnnotation("cluster2")),
newUnstructuredObject(podGVK, "pod22", withCacheSourceAnnotation("cluster2")),
}
if !reflect.DeepEqual(items, expect) {
t.Errorf("list items diff: %v", cmp.Diff(expect, items))
}
}
func TestMultiClusterCache_List_Namespaced(t *testing.T) {
cluster1 := newCluster("cluster1")
cluster2 := newCluster("cluster2")
cluster1Client := fakedynamic.NewSimpleDynamicClient(scheme,
newUnstructuredObject(podGVK, "pod11", withNamespace("ns1"), withResourceVersion("1000")),
newUnstructuredObject(podGVK, "pod12", withNamespace("ns2"), withResourceVersion("1000")),
)
cluster2Client := fakedynamic.NewSimpleDynamicClient(scheme,
newUnstructuredObject(podGVK, "pod21", withNamespace("ns1"), withResourceVersion("2000")),
newUnstructuredObject(podGVK, "pod22", withNamespace("ns2"), withResourceVersion("2000")),
)
newClientFunc := func(cluster string) (dynamic.Interface, error) {
switch cluster {
case cluster1.Name:
return cluster1Client, nil
case cluster2.Name:
return cluster2Client, nil
}
return fakedynamic.NewSimpleDynamicClient(scheme), nil
}
cache := NewMultiClusterCache(newClientFunc, restMapper)
defer cache.Stop()
err := cache.UpdateCache(map[string]map[schema.GroupVersionResource]*MultiNamespace{
cluster1.Name: {podGVR: &MultiNamespace{namespaces: sets.New[string]("ns1")}},
cluster2.Name: {podGVR: &MultiNamespace{namespaces: sets.New[string]("ns1", "ns2", "ns3")}},
})
if err != nil {
t.Error(err)
return
}
type args struct {
ns string
gvr schema.GroupVersionResource
options *metainternalversion.ListOptions
}
type want struct {
names sets.Set[string]
errAssert func(error) bool
}
tests := []struct {
name string
args args
want want
}{
{
name: "request all namespaces",
args: args{
ns: metav1.NamespaceAll,
gvr: podGVR,
options: &metainternalversion.ListOptions{},
},
want: want{
names: sets.New[string]("pod11", "pod21", "pod22"),
errAssert: noError,
},
},
{
name: "request ns1 namespaces",
args: args{
ns: "ns1",
gvr: podGVR,
options: &metainternalversion.ListOptions{},
},
want: want{
names: sets.New[string]("pod11", "pod21"),
errAssert: noError,
},
},
{
name: "request ns2 namespaces",
args: args{
ns: "ns2",
gvr: podGVR,
options: &metainternalversion.ListOptions{},
},
want: want{
names: sets.New[string]("pod22"),
errAssert: noError,
},
},
{
name: "request ns3 namespaces",
args: args{
ns: "ns3",
gvr: podGVR,
options: &metainternalversion.ListOptions{},
},
want: want{
names: sets.New[string](),
errAssert: noError,
},
},
{
name: "request ns4 namespaces",
args: args{
ns: "ns4",
gvr: podGVR,
options: &metainternalversion.ListOptions{},
},
want: want{
names: sets.New[string](),
errAssert: noError,
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
obj, err := cache.List(request.WithNamespace(context.TODO(), tt.args.ns), tt.args.gvr, tt.args.options)
if !tt.want.errAssert(err) {
t.Errorf("Unexpected error: %v", err)
return
}
if err != nil {
return
}
names := sets.New[string]()
err = meta.EachListItem(obj, func(o runtime.Object) error {
a, err := meta.Accessor(o)
if err != nil {
return err
}
names.Insert(a.GetName())
return nil
})
if err != nil {
t.Error(err)
return
}
if !tt.want.names.Equal(names) {
t.Errorf("List items want=%v, actual=%v", strings.Join(sets.List(tt.want.names), ","), strings.Join(sets.List(names), ","))
}
})
}
}
func TestMultiClusterCache_Watch(t *testing.T) {
cluster1 := newCluster("cluster1")
cluster2 := newCluster("cluster2")
cluster1Client := NewEnhancedFakeDynamicClientWithResourceVersion(scheme, "1002",
newUnstructuredObject(podGVK, "pod11", withDefaultNamespace(), withResourceVersion("1001")),
newUnstructuredObject(podGVK, "pod12", withDefaultNamespace(), withResourceVersion("1002")),
)
cluster2Client := NewEnhancedFakeDynamicClientWithResourceVersion(scheme, "2002",
newUnstructuredObject(podGVK, "pod21", withDefaultNamespace(), withResourceVersion("2001")),
newUnstructuredObject(podGVK, "pod22", withDefaultNamespace(), withResourceVersion("2002")),
)
newClientFunc := func(cluster string) (dynamic.Interface, error) {
switch cluster {
case cluster1.Name:
return cluster1Client, nil
case cluster2.Name:
return cluster2Client, nil
}
return fakedynamic.NewSimpleDynamicClient(scheme), nil
}
cache := NewMultiClusterCache(newClientFunc, restMapper)
defer cache.Stop()
err := cache.UpdateCache(map[string]map[schema.GroupVersionResource]*MultiNamespace{
cluster1.Name: resourceSet(podGVR),
cluster2.Name: resourceSet(podGVR),
})
if err != nil {
t.Error(err)
return
}
// wait cache synced
time.Sleep(time.Second)
// put gets into Cacher.incoming chan
_ = cluster1Client.Tracker().Add(newUnstructuredObject(podGVK, "pod13", withDefaultNamespace(), withResourceVersion("1003")))
_ = cluster2Client.Tracker().Add(newUnstructuredObject(podGVK, "pod23", withDefaultNamespace(), withResourceVersion("2003")))
cluster1Client.versionTracker.Set("1003")
cluster2Client.versionTracker.Set("2003")
type args struct {
options *metainternalversion.ListOptions
}
type want struct {
gets sets.Set[string]
}
tests := []struct {
name string
args args
want want
}{
{
name: "resource version is zero",
args: args{
options: &metainternalversion.ListOptions{
ResourceVersion: "0",
},
},
want: want{
gets: sets.New[string]("pod11", "pod12", "pod13", "pod21", "pod22", "pod23"),
},
},
{
name: "resource version of cluster2 is zero",
args: args{
options: &metainternalversion.ListOptions{
ResourceVersion: buildMultiClusterRV(cluster1.Name, "1002", cluster2.Name, "0"),
},
},
want: want{
gets: sets.New[string]("pod13", "pod21", "pod22", "pod23"),
},
},
{
name: "resource versions are not zero",
args: args{
options: &metainternalversion.ListOptions{
ResourceVersion: buildMultiClusterRV(cluster1.Name, "1002", cluster2.Name, "2002"),
},
},
want: want{
gets: sets.New[string]("pod13", "pod23"),
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
ctx, cancel := context.WithCancel(request.WithNamespace(context.TODO(), metav1.NamespaceDefault))
defer cancel()
watcher, err := cache.Watch(ctx, podGVR, tt.args.options)
if err != nil {
t.Error(err)
return
}
defer watcher.Stop()
waitCh := time.After(time.Second)
gets := sets.New[string]()
LOOP:
for {
select {
case event, ok := <-watcher.ResultChan():
if !ok {
break LOOP
}
accessor, err := meta.Accessor(event.Object)
if err == nil {
gets.Insert(accessor.GetName())
}
case <-waitCh:
break LOOP
}
}
if !tt.want.gets.Equal(gets) {
t.Errorf("Watch() got = %v, but want = %v", gets, tt.want.gets)
}
})
}
}
func TestMultiClusterCache_Watch_Namespaced(t *testing.T) {
cluster1 := newCluster("cluster1")
cluster2 := newCluster("cluster2")
cluster1Client := NewEnhancedFakeDynamicClientWithResourceVersion(scheme, "1002",
newUnstructuredObject(podGVK, "pod11", withNamespace("ns1"), withResourceVersion("1001")),
newUnstructuredObject(podGVK, "pod12", withNamespace("ns2"), withResourceVersion("1002")),
)
cluster2Client := NewEnhancedFakeDynamicClientWithResourceVersion(scheme, "2002",
newUnstructuredObject(podGVK, "pod21", withNamespace("ns1"), withResourceVersion("2001")),
newUnstructuredObject(podGVK, "pod22", withNamespace("ns2"), withResourceVersion("2002")),
)
newClientFunc := func(cluster string) (dynamic.Interface, error) {
switch cluster {
case cluster1.Name:
return cluster1Client, nil
case cluster2.Name:
return cluster2Client, nil
}
return fakedynamic.NewSimpleDynamicClient(scheme), nil
}
cache := NewMultiClusterCache(newClientFunc, restMapper)
defer cache.Stop()
err := cache.UpdateCache(map[string]map[schema.GroupVersionResource]*MultiNamespace{
cluster1.Name: {podGVR: &MultiNamespace{namespaces: sets.New[string]("ns1")}},
cluster2.Name: {podGVR: &MultiNamespace{namespaces: sets.New[string]("ns1", "ns2", "ns3")}},
})
if err != nil {
t.Error(err)
return
}
// wait cache synced
time.Sleep(time.Second)
type args struct {
ns string
}
type want struct {
names sets.Set[string]
errAssert func(error) bool
}
tests := []struct {
name string
args args
want want
}{
{
name: "request all namespaces",
args: args{
ns: metav1.NamespaceAll,
},
want: want{
names: sets.New[string]("pod11", "pod21", "pod22"),
errAssert: noError,
},
},
{
name: "request ns1 namespace",
args: args{
ns: "ns1",
},
want: want{
names: sets.New[string]("pod11", "pod21"),
errAssert: noError,
},
},
{
name: "request ns2 namespace",
args: args{
ns: "ns2",
},
want: want{
names: sets.New[string]("pod22"),
errAssert: noError,
},
},
{
name: "request ns3 namespace",
args: args{
ns: "ns3",
},
want: want{
names: sets.New[string](),
errAssert: noError,
},
},
{
name: "request ns4 namespace",
args: args{
ns: "ns4",
},
want: want{
names: sets.New[string](),
errAssert: noError,
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
ctx, cancel := context.WithCancel(request.WithNamespace(context.TODO(), tt.args.ns))
defer cancel()
watcher, err := cache.Watch(ctx, podGVR, &metainternalversion.ListOptions{ResourceVersion: "0"})
if err != nil {
t.Error(err)
return
}
defer watcher.Stop()
waitCh := time.After(time.Second)
gots := sets.New[string]()
LOOP:
for {
select {
case event, ok := <-watcher.ResultChan():
if !ok {
break LOOP
}
accessor, err := meta.Accessor(event.Object)
if err == nil {
gots.Insert(accessor.GetName())
}
case <-waitCh:
break LOOP
}
}
if !tt.want.names.Equal(gots) {
t.Errorf("Watch() got = %v, but want = %v", gots, tt.want.names)
}
})
}
}
func newCluster(name string) *clusterv1alpha1.Cluster {
o := &clusterv1alpha1.Cluster{}
o.Name = name
return o
}
func newUnstructuredObject(gvk schema.GroupVersionKind, name string, options ...func(object *unstructured.Unstructured)) *unstructured.Unstructured {
obj := &unstructured.Unstructured{}
obj.SetGroupVersionKind(gvk)
obj.SetName(name)
for _, opt := range options {
opt(obj)
}
return obj
}
func withDefaultNamespace() func(*unstructured.Unstructured) {
return func(obj *unstructured.Unstructured) {
obj.SetNamespace(metav1.NamespaceDefault)
}
}
func withNamespace(ns string) func(*unstructured.Unstructured) {
return func(obj *unstructured.Unstructured) {
obj.SetNamespace(ns)
}
}
func withResourceVersion(rv string) func(*unstructured.Unstructured) {
return func(obj *unstructured.Unstructured) {
obj.SetResourceVersion(rv)
}
}
func withCacheSourceAnnotation(cluster string) func(*unstructured.Unstructured) {
return func(obj *unstructured.Unstructured) {
addCacheSourceAnnotation(obj, cluster)
}
}
func withLabel(label, value string) func(*unstructured.Unstructured) {
return func(obj *unstructured.Unstructured) {
err := unstructured.SetNestedField(obj.Object, value, "metadata", "labels", label)
if err != nil {
panic(err)
}
}
}
func noError(err error) bool {
return err == nil
}
func buildMultiClusterRV(clusterAndRV ...string) string {
m := newMultiClusterResourceVersionWithCapacity(len(clusterAndRV) / 2)
for i := 0; i < len(clusterAndRV); {
m.set(clusterAndRV[i], clusterAndRV[i+1])
i += 2
}
return m.String()
}
func asLabelSelector(s string) labels.Selector {
selector, err := labels.Parse(s)
if err != nil {
panic(fmt.Sprintf("Fail to parse %s to labels: %v", s, err))
}
return selector
}
func resourceSet(rs ...schema.GroupVersionResource) map[schema.GroupVersionResource]*MultiNamespace {
m := make(map[schema.GroupVersionResource]*MultiNamespace, len(rs))
for _, r := range rs {
m[r] = &MultiNamespace{allNamespaces: true}
}
return m
}
type VersionTracker interface {
Set(string)
Get() string
}
type versionTracker struct {
lock sync.RWMutex
rv string
}
func NewVersionTracker(rv string) VersionTracker {
return &versionTracker{
rv: rv,
}
}
func (t *versionTracker) Set(rv string) {
t.lock.Lock()
defer t.lock.Unlock()
t.rv = rv
}
func (t *versionTracker) Get() string {
t.lock.RLock()
defer t.lock.RUnlock()
return t.rv
}
// EnhancedFakeDynamicClient enhances FakeDynamicClient. It will return resourceVersion for list request.
type EnhancedFakeDynamicClient struct {
*fakedynamic.FakeDynamicClient
ObjectReaction kubetesting.ReactionFunc
versionTracker VersionTracker
}
// NewEnhancedFakeDynamicClientWithResourceVersion returns instance of EnhancedFakeDynamicClient.
func NewEnhancedFakeDynamicClientWithResourceVersion(scheme *runtime.Scheme, rv string, objects ...runtime.Object) *EnhancedFakeDynamicClient {
v := NewVersionTracker(rv)
c := fakedynamic.NewSimpleDynamicClient(scheme, objects...)
c.PrependReactor("list", "*", enhancedListReaction(c.Tracker(), v))
return &EnhancedFakeDynamicClient{
FakeDynamicClient: c,
ObjectReaction: kubetesting.ObjectReaction(c.Tracker()),
versionTracker: v,
}
}
func enhancedListReaction(o kubetesting.ObjectTracker, v VersionTracker) kubetesting.ReactionFunc {
return func(act kubetesting.Action) (bool, runtime.Object, error) {
action, ok := act.(kubetesting.ListActionImpl)
if !ok {
return false, nil, nil
}
ret, err := o.List(action.GetResource(), action.GetKind(), action.GetNamespace())
if err != nil {
return true, ret, err
}
accessor, err := meta.ListAccessor(ret)
if err != nil {
// object is not a list object, don't change it. Don't return this error.
return true, ret, nil
}
accessor.SetResourceVersion(v.Get())
return true, ret, nil
}
}
func Test_prepareBeforeList(t *testing.T) {
testMRV := &multiClusterResourceVersion{rvs: map[string]string{"c1": "1234", "c2": "5678"}}
testMRVString := testMRV.String()
type args struct {
o *metainternalversion.ListOptions
}
tests := []struct {
name string
args args
wantCluster string
wantOptions *metainternalversion.ListOptions
wantMrv *multiClusterResourceVersion
}{
{
name: "Continue is empty",
args: args{
o: &metainternalversion.ListOptions{
ResourceVersion: "0",
},
},
wantCluster: "",
wantOptions: &metainternalversion.ListOptions{
ResourceVersion: "0",
Continue: "",
},
wantMrv: newMultiClusterResourceVersionFromString("0"),
},
{
name: "Continue is not empty, and has no continue",
args: args{
o: &metainternalversion.ListOptions{
Continue: func() string {
c := multiClusterContinue{RV: testMRVString, Cluster: "c2", Continue: ""}
return c.String()
}(),
},
},
wantCluster: "c2",
wantOptions: &metainternalversion.ListOptions{
ResourceVersion: "5678",
Continue: "",
},
wantMrv: testMRV,
},
{
name: "Continue is not empty, and has continue",
args: args{
o: &metainternalversion.ListOptions{
Continue: func() string {
c := multiClusterContinue{RV: testMRVString, Cluster: "c2", Continue: "xxx"}
return c.String()
}(),
},
},
wantCluster: "c2",
wantOptions: &metainternalversion.ListOptions{
Continue: "xxx",
},
wantMrv: testMRV,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
gotCluster, gotOptions, gotMrv := prepareBeforeList(tt.args.o)
if gotCluster != tt.wantCluster {
t.Errorf("prepareBeforeList() gotCluster = %v, want %v", gotCluster, tt.wantCluster)
}
if !reflect.DeepEqual(gotOptions, tt.wantOptions) {
t.Errorf("prepareBeforeList() gotOptions = %v, want %v", gotOptions, tt.wantOptions)
}
if !reflect.DeepEqual(gotMrv, tt.wantMrv) {
t.Errorf("prepareBeforeList() gotMrv = %v, want %v", gotMrv, tt.wantMrv)
}
})
}
}
func TestMultiClusterCache_fillMissingClusterResourceVersion(t *testing.T) {
cluster1 := newCluster("cluster1")
cluster1Client := NewEnhancedFakeDynamicClientWithResourceVersion(scheme, "1000")
cluster1Client.AddReactor("*", "nodes", func(kubetesting.Action) (bool, runtime.Object, error) {
return false, nil, fmt.Errorf("error")
})
newClientFunc := func(cluster string) (dynamic.Interface, error) {
switch cluster {
case cluster1.Name:
return cluster1Client, nil
default:
return nil, fmt.Errorf("unknown cluster %v", cluster)
}
}
cache := NewMultiClusterCache(newClientFunc, restMapper)
defer cache.Stop()
err := cache.UpdateCache(map[string]map[schema.GroupVersionResource]*MultiNamespace{
cluster1.Name: resourceSet(podGVR),
})
if err != nil {
t.Fatal(err)
}
// wait cache synced
time.Sleep(time.Second)
type args struct {
mcv *multiClusterResourceVersion
clusters []string
}
tests := []struct {
name string
args args
wantErr bool
wantMCV *multiClusterResourceVersion
}{
{
name: "no missing cluster rv",
args: args{
mcv: &multiClusterResourceVersion{
rvs: map[string]string{
"cluster1": "1234",
},
},
clusters: []string{"cluster1"},
},
wantMCV: &multiClusterResourceVersion{
rvs: map[string]string{
"cluster1": "1234",
},
},
},
{
name: "has missing cluster rv",
args: args{
mcv: &multiClusterResourceVersion{
rvs: map[string]string{},
},
clusters: []string{"cluster1"},
},
wantMCV: &multiClusterResourceVersion{
rvs: map[string]string{
"cluster1": "1000",
},
},
},
{
name: "cluster not exist",
args: args{
mcv: &multiClusterResourceVersion{
rvs: map[string]string{},
},
clusters: []string{"non-exist"},
},
wantMCV: &multiClusterResourceVersion{
rvs: map[string]string{},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
defer cancel()
if err = cache.fillMissingClusterResourceVersion(ctx, tt.args.mcv, tt.args.clusters, podGVR); (err != nil) != tt.wantErr {
t.Fatalf("fillMissingClusterResourceVersion() error = %v, wantErr %v", err, tt.wantErr)
}
if got := tt.args.mcv; !reflect.DeepEqual(got, tt.wantMCV) {
t.Fatalf("fillMissingClusterResourceVersion() mcv = %v, want %v", got.String(), tt.wantMCV.String())
}
})
}
}