mirror of https://github.com/grpc/grpc-go.git
315 lines
14 KiB
Go
315 lines
14 KiB
Go
/*
|
|
*
|
|
* Copyright 2022 gRPC authors.
|
|
*
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
* you may not use this file except in compliance with the License.
|
|
* You may obtain a copy of the License at
|
|
*
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
* See the License for the specific language governing permissions and
|
|
* limitations under the License.
|
|
*
|
|
*/
|
|
|
|
package xdsclient_test
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/google/uuid"
|
|
"google.golang.org/grpc/internal/testutils"
|
|
"google.golang.org/grpc/internal/testutils/xds/e2e"
|
|
"google.golang.org/grpc/internal/xds/bootstrap"
|
|
xdstestutils "google.golang.org/grpc/xds/internal/testutils"
|
|
"google.golang.org/grpc/xds/internal/xdsclient"
|
|
"google.golang.org/grpc/xds/internal/xdsclient/xdsresource"
|
|
|
|
v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
|
|
)
|
|
|
|
const (
|
|
testAuthority1 = "test-authority1"
|
|
testAuthority2 = "test-authority2"
|
|
testAuthority3 = "test-authority3"
|
|
)
|
|
|
|
var (
|
|
// These two resources use `testAuthority1`, which contains an empty server
|
|
// config in the bootstrap file, and therefore will use the default
|
|
// management server.
|
|
authorityTestResourceName11 = xdstestutils.BuildResourceName(xdsresource.ClusterResourceTypeName, testAuthority1, cdsName+"1", nil)
|
|
authorityTestResourceName12 = xdstestutils.BuildResourceName(xdsresource.ClusterResourceTypeName, testAuthority1, cdsName+"2", nil)
|
|
// This resource uses `testAuthority2`, which contains an empty server
|
|
// config in the bootstrap file, and therefore will use the default
|
|
// management server.
|
|
authorityTestResourceName2 = xdstestutils.BuildResourceName(xdsresource.ClusterResourceTypeName, testAuthority2, cdsName+"3", nil)
|
|
// This resource uses `testAuthority3`, which contains a non-empty server
|
|
// config in the bootstrap file, and therefore will use the non-default
|
|
// management server.
|
|
authorityTestResourceName3 = xdstestutils.BuildResourceName(xdsresource.ClusterResourceTypeName, testAuthority3, cdsName+"3", nil)
|
|
)
|
|
|
|
// setupForAuthorityTests spins up two management servers, one to act as the
|
|
// default and the other to act as the non-default. It also generates a
|
|
// bootstrap configuration with three authorities (the first two pointing to the
|
|
// default and the third one pointing to the non-default).
|
|
//
|
|
// Returns two listeners used by the default and non-default management servers
|
|
// respectively, and the xDS client and its close function.
|
|
func setupForAuthorityTests(ctx context.Context, t *testing.T, idleTimeout time.Duration) (*testutils.ListenerWrapper, *testutils.ListenerWrapper, xdsclient.XDSClient, func()) {
|
|
// Create listener wrappers which notify on to a channel whenever a new
|
|
// connection is accepted. We use this to track the number of transports
|
|
// used by the xDS client.
|
|
lisDefault := testutils.NewListenerWrapper(t, nil)
|
|
lisNonDefault := testutils.NewListenerWrapper(t, nil)
|
|
|
|
// Start a management server to act as the default authority.
|
|
defaultAuthorityServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: lisDefault})
|
|
|
|
// Start a management server to act as the non-default authority.
|
|
nonDefaultAuthorityServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: lisNonDefault})
|
|
|
|
// Create a bootstrap configuration with two non-default authorities which
|
|
// have empty server configs, and therefore end up using the default server
|
|
// config, which points to the above management server.
|
|
nodeID := uuid.New().String()
|
|
bootstrapContents, err := bootstrap.NewContentsForTesting(bootstrap.ConfigOptionsForTesting{
|
|
Servers: []byte(fmt.Sprintf(`[{
|
|
"server_uri": %q,
|
|
"channel_creds": [{"type": "insecure"}]
|
|
}]`, defaultAuthorityServer.Address)),
|
|
Node: []byte(fmt.Sprintf(`{"id": "%s"}`, nodeID)),
|
|
Authorities: map[string]json.RawMessage{
|
|
testAuthority1: []byte(`{}`),
|
|
testAuthority2: []byte(`{}`),
|
|
testAuthority3: []byte(fmt.Sprintf(`{
|
|
"xds_servers": [{
|
|
"server_uri": %q,
|
|
"channel_creds": [{"type": "insecure"}]
|
|
}]}`, nonDefaultAuthorityServer.Address)),
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("Failed to create bootstrap configuration: %v", err)
|
|
}
|
|
client, close, err := xdsclient.NewForTesting(xdsclient.OptionsForTesting{
|
|
Name: t.Name(),
|
|
Contents: bootstrapContents,
|
|
WatchExpiryTimeout: defaultTestWatchExpiryTimeout,
|
|
AuthorityIdleTimeout: idleTimeout,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("Failed to create an xDS client: %v", err)
|
|
}
|
|
|
|
resources := e2e.UpdateOptions{
|
|
NodeID: nodeID,
|
|
Clusters: []*v3clusterpb.Cluster{
|
|
e2e.DefaultCluster(authorityTestResourceName11, edsName, e2e.SecurityLevelNone),
|
|
e2e.DefaultCluster(authorityTestResourceName12, edsName, e2e.SecurityLevelNone),
|
|
e2e.DefaultCluster(authorityTestResourceName2, edsName, e2e.SecurityLevelNone),
|
|
e2e.DefaultCluster(authorityTestResourceName3, edsName, e2e.SecurityLevelNone),
|
|
},
|
|
SkipValidation: true,
|
|
}
|
|
if err := defaultAuthorityServer.Update(ctx, resources); err != nil {
|
|
t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err)
|
|
}
|
|
return lisDefault, lisNonDefault, client, close
|
|
}
|
|
|
|
// TestAuthorityShare tests the authority sharing logic. The test verifies the
|
|
// following scenarios:
|
|
// - A watch for a resource name with an authority matching an existing watch
|
|
// should not result in a new transport being created.
|
|
// - A watch for a resource name with different authority name but same
|
|
// authority config as an existing watch should not result in a new transport
|
|
// being created.
|
|
func (s) TestAuthorityShare(t *testing.T) {
|
|
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
|
|
defer cancel()
|
|
lis, _, client, close := setupForAuthorityTests(ctx, t, time.Duration(0))
|
|
defer close()
|
|
|
|
// Verify that no connection is established to the management server at this
|
|
// point. A transport is created only when a resource (which belongs to that
|
|
// authority) is requested.
|
|
sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
|
|
defer sCancel()
|
|
if _, err := lis.NewConnCh.Receive(sCtx); err != context.DeadlineExceeded {
|
|
t.Fatal("Unexpected new transport created to management server")
|
|
}
|
|
|
|
// Request the first resource. Verify that a new transport is created.
|
|
watcher := noopClusterWatcher{}
|
|
cdsCancel1 := xdsresource.WatchCluster(client, authorityTestResourceName11, watcher)
|
|
defer cdsCancel1()
|
|
if _, err := lis.NewConnCh.Receive(ctx); err != nil {
|
|
t.Fatalf("Timed out when waiting for a new transport to be created to the management server: %v", err)
|
|
}
|
|
|
|
// Request the second resource. Verify that no new transport is created.
|
|
cdsCancel2 := xdsresource.WatchCluster(client, authorityTestResourceName12, watcher)
|
|
defer cdsCancel2()
|
|
sCtx, sCancel = context.WithTimeout(ctx, defaultTestShortTimeout)
|
|
defer sCancel()
|
|
if _, err := lis.NewConnCh.Receive(sCtx); err != context.DeadlineExceeded {
|
|
t.Fatal("Unexpected new transport created to management server")
|
|
}
|
|
|
|
// Request the third resource. Verify that no new transport is created.
|
|
cdsCancel3 := xdsresource.WatchCluster(client, authorityTestResourceName2, watcher)
|
|
defer cdsCancel3()
|
|
sCtx, sCancel = context.WithTimeout(ctx, defaultTestShortTimeout)
|
|
defer sCancel()
|
|
if _, err := lis.NewConnCh.Receive(sCtx); err != context.DeadlineExceeded {
|
|
t.Fatal("Unexpected new transport created to management server")
|
|
}
|
|
}
|
|
|
|
// TestAuthorityIdle test the authority idle timeout logic. The test verifies
|
|
// that the xDS client does not close authorities immediately after the last
|
|
// watch is canceled, but waits for the configured idle timeout to expire before
|
|
// closing them.
|
|
func (s) TestAuthorityIdleTimeout(t *testing.T) {
|
|
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
|
|
defer cancel()
|
|
lis, _, client, close := setupForAuthorityTests(ctx, t, defaultTestIdleAuthorityTimeout)
|
|
defer close()
|
|
|
|
// Request the first resource. Verify that a new transport is created.
|
|
watcher := noopClusterWatcher{}
|
|
cdsCancel1 := xdsresource.WatchCluster(client, authorityTestResourceName11, watcher)
|
|
val, err := lis.NewConnCh.Receive(ctx)
|
|
if err != nil {
|
|
t.Fatalf("Timed out when waiting for a new transport to be created to the management server: %v", err)
|
|
}
|
|
conn := val.(*testutils.ConnWrapper)
|
|
|
|
// Request the second resource. Verify that no new transport is created.
|
|
cdsCancel2 := xdsresource.WatchCluster(client, authorityTestResourceName12, watcher)
|
|
sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
|
|
defer sCancel()
|
|
if _, err := lis.NewConnCh.Receive(sCtx); err != context.DeadlineExceeded {
|
|
t.Fatal("Unexpected new transport created to management server")
|
|
}
|
|
|
|
// Cancel both watches, and verify that the connection to the management
|
|
// server is not closed immediately.
|
|
cdsCancel1()
|
|
cdsCancel2()
|
|
sCtx, sCancel = context.WithTimeout(ctx, defaultTestShortTimeout)
|
|
defer sCancel()
|
|
if _, err := conn.CloseCh.Receive(sCtx); err != context.DeadlineExceeded {
|
|
t.Fatal("Connection to management server closed unexpectedly")
|
|
}
|
|
|
|
// Wait for the authority idle timeout to fire.
|
|
time.Sleep(2 * defaultTestIdleAuthorityTimeout)
|
|
sCtx, sCancel = context.WithTimeout(ctx, defaultTestShortTimeout)
|
|
defer sCancel()
|
|
if _, err := conn.CloseCh.Receive(sCtx); err != nil {
|
|
t.Fatal("Connection to management server not closed after idle timeout expiry")
|
|
}
|
|
}
|
|
|
|
// TestAuthorityClientClose verifies that authorities in use and in the idle
|
|
// cache are all closed when the client is closed.
|
|
func (s) TestAuthorityClientClose(t *testing.T) {
|
|
// Set the authority idle timeout to twice the defaultTestTimeout. This will
|
|
// ensure that idle authorities stay in the cache for the duration of this
|
|
// test, until explicitly closed.
|
|
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
|
|
defer cancel()
|
|
lisDefault, lisNonDefault, client, close := setupForAuthorityTests(ctx, t, time.Duration(2*defaultTestTimeout))
|
|
|
|
// Request the first resource. Verify that a new transport is created to the
|
|
// default management server.
|
|
watcher := noopClusterWatcher{}
|
|
cdsCancel1 := xdsresource.WatchCluster(client, authorityTestResourceName11, watcher)
|
|
val, err := lisDefault.NewConnCh.Receive(ctx)
|
|
if err != nil {
|
|
t.Fatalf("Timed out when waiting for a new transport to be created to the management server: %v", err)
|
|
}
|
|
connDefault := val.(*testutils.ConnWrapper)
|
|
|
|
// Request another resource which is served by the non-default authority.
|
|
// Verify that a new transport is created to the non-default management
|
|
// server.
|
|
xdsresource.WatchCluster(client, authorityTestResourceName3, watcher)
|
|
val, err = lisNonDefault.NewConnCh.Receive(ctx)
|
|
if err != nil {
|
|
t.Fatalf("Timed out when waiting for a new transport to be created to the management server: %v", err)
|
|
}
|
|
connNonDefault := val.(*testutils.ConnWrapper)
|
|
|
|
// Cancel the first watch. This should move the default authority to the
|
|
// idle cache, but the connection should not be closed yet, because the idle
|
|
// timeout would not have fired.
|
|
cdsCancel1()
|
|
sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
|
|
defer sCancel()
|
|
if _, err := connDefault.CloseCh.Receive(sCtx); err != context.DeadlineExceeded {
|
|
t.Fatal("Connection to management server closed unexpectedly")
|
|
}
|
|
|
|
// Closing the xDS client should close the connection to both management
|
|
// servers, even though we have an open watch to one of them.
|
|
close()
|
|
if _, err := connDefault.CloseCh.Receive(ctx); err != nil {
|
|
t.Fatal("Connection to management server not closed after client close")
|
|
}
|
|
if _, err := connNonDefault.CloseCh.Receive(ctx); err != nil {
|
|
t.Fatal("Connection to management server not closed after client close")
|
|
}
|
|
}
|
|
|
|
// TestAuthorityRevive verifies that an authority in the idle cache is revived
|
|
// when a new watch is started on this authority.
|
|
func (s) TestAuthorityRevive(t *testing.T) {
|
|
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
|
|
defer cancel()
|
|
lis, _, client, close := setupForAuthorityTests(ctx, t, defaultTestIdleAuthorityTimeout)
|
|
defer close()
|
|
|
|
// Request the first resource. Verify that a new transport is created.
|
|
watcher := noopClusterWatcher{}
|
|
cdsCancel1 := xdsresource.WatchCluster(client, authorityTestResourceName11, watcher)
|
|
val, err := lis.NewConnCh.Receive(ctx)
|
|
if err != nil {
|
|
t.Fatalf("Timed out when waiting for a new transport to be created to the management server: %v", err)
|
|
}
|
|
conn := val.(*testutils.ConnWrapper)
|
|
|
|
// Cancel the above watch. This should move the authority to the idle cache.
|
|
cdsCancel1()
|
|
|
|
// Request the second resource. Verify that no new transport is created.
|
|
// This should move the authority out of the idle cache.
|
|
cdsCancel2 := xdsresource.WatchCluster(client, authorityTestResourceName12, watcher)
|
|
defer cdsCancel2()
|
|
sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
|
|
defer sCancel()
|
|
if _, err := lis.NewConnCh.Receive(sCtx); err != context.DeadlineExceeded {
|
|
t.Fatal("Unexpected new transport created to management server")
|
|
}
|
|
|
|
// Wait for double the idle timeout, and the connection to the management
|
|
// server should not be closed, since it was revived from the idle cache.
|
|
time.Sleep(2 * defaultTestIdleAuthorityTimeout)
|
|
sCtx, sCancel = context.WithTimeout(ctx, defaultTestShortTimeout)
|
|
defer sCancel()
|
|
if _, err := conn.CloseCh.Receive(sCtx); err != context.DeadlineExceeded {
|
|
t.Fatal("Connection to management server closed unexpectedly")
|
|
}
|
|
}
|