test(multicluster): Wait for cluster store to be populated in test

The federated service watcher test has a race condition where we create a cluster store with a set of kubernetes manifests and then immediately begin testing queries to that cluster store.  If these queries are executed before the cluster store's informers process the kubernetes manifests, the queries can fail.

In the context of this test, this failure manifests as the read on the updates channel never returning, resulting in test timeouts.

We fix this by waiting for the cluster store to be populated before continuing with the test and issuing queries.

Signed-off-by: Alex Leong <alex@buoyant.io>

controller/api/destination/federated_service_watcher_test.go

@@ -1,9 +1,11 @@
 package destination
 
 import (
+	"errors"
 	"fmt"
 	"slices"
 	"testing"
+	"time"
 
 	logging "github.com/sirupsen/logrus"
 
@@ -11,6 +13,7 @@ import (
 	"github.com/linkerd/linkerd2/controller/api/destination/watcher"
 	"github.com/linkerd/linkerd2/controller/k8s"
 	"github.com/linkerd/linkerd2/pkg/addr"
+	"github.com/linkerd/linkerd2/testutil"
 	"github.com/prometheus/client_golang/prometheus"
 )
 
@@ -157,6 +160,20 @@ func mockFederatedServiceWatcher(t *testing.T) (*federatedServiceWatcher, error)
 	metadataAPI.Sync(nil)
 	clusterStore.Sync(nil)
 
+	// Wait for the cluster store to be populated with the remote clusters.
+	err = testutil.RetryFor(30*time.Second, func() error {
+		if _, _, found := clusterStore.Get("east"); !found {
+			return errors.New("east cluster not found in cluster store")
+		}
+		if _, _, found := clusterStore.Get("north"); !found {
+			return errors.New("north cluster not found in cluster store")
+		}
+		return nil
+	})
+	if err != nil {
+		return nil, fmt.Errorf("timed out waiting for cluster store to be populated: %w", err)
+	}
+
 	return fsw, nil
 }