mirror of https://github.com/grpc/grpc-go.git
335 lines
14 KiB
Go
335 lines
14 KiB
Go
/*
|
|
*
|
|
* Copyright 2022 gRPC authors.
|
|
*
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
* you may not use this file except in compliance with the License.
|
|
* You may obtain a copy of the License at
|
|
*
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
* See the License for the specific language governing permissions and
|
|
* limitations under the License.
|
|
*
|
|
*/
|
|
|
|
package xdsclient_test
|
|
|
|
import (
|
|
"context"
|
|
"net"
|
|
"testing"
|
|
|
|
"github.com/google/uuid"
|
|
"google.golang.org/grpc/credentials/insecure"
|
|
"google.golang.org/grpc/xds/internal/clients"
|
|
"google.golang.org/grpc/xds/internal/clients/grpctransport"
|
|
"google.golang.org/grpc/xds/internal/clients/internal/testutils"
|
|
"google.golang.org/grpc/xds/internal/clients/internal/testutils/e2e"
|
|
"google.golang.org/grpc/xds/internal/clients/xdsclient"
|
|
"google.golang.org/grpc/xds/internal/clients/xdsclient/internal/xdsresource"
|
|
|
|
v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
|
|
)
|
|
|
|
const (
|
|
testAuthority1 = "test-authority1"
|
|
testAuthority2 = "test-authority2"
|
|
testAuthority3 = "test-authority3"
|
|
)
|
|
|
|
var (
|
|
// These two resources use `testAuthority1`, which contains an empty server
|
|
// config and therefore will use the default management server.
|
|
authorityTestResourceName11 = buildResourceName(listenerResourceTypeName, testAuthority1, ldsName, nil)
|
|
authorityTestResourceName12 = buildResourceName(listenerResourceTypeName, testAuthority1, ldsName+"2", nil)
|
|
// This resource uses `testAuthority2`, which contains an empty server
|
|
// config and therefore will use the default management server.
|
|
authorityTestResourceName2 = buildResourceName(listenerResourceTypeName, testAuthority2, ldsName+"3", nil)
|
|
// This resource uses `testAuthority3`, which contains a non-empty server
|
|
// config, and therefore will use the non-default management server.
|
|
authorityTestResourceName3 = buildResourceName(listenerResourceTypeName, testAuthority3, ldsName+"3", nil)
|
|
)
|
|
|
|
// setupForAuthorityTests spins up two management servers, one to act as the
|
|
// default and the other to act as the non-default. It also creates a
|
|
// xDS client configuration with three authorities (the first two pointing to
|
|
// the default and the third one pointing to the non-default).
|
|
//
|
|
// Returns two listeners used by the default and non-default management servers
|
|
// respectively, and the xDS client.
|
|
func setupForAuthorityTests(ctx context.Context, t *testing.T) (*testutils.ListenerWrapper, *testutils.ListenerWrapper, *xdsclient.XDSClient) {
|
|
// Create listener wrappers which notify on to a channel whenever a new
|
|
// connection is accepted. We use this to track the number of transports
|
|
// used by the xDS client.
|
|
lisDefault := testutils.NewListenerWrapper(t, nil)
|
|
lisNonDefault := testutils.NewListenerWrapper(t, nil)
|
|
|
|
// Start a management server to act as the default authority.
|
|
defaultAuthorityServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: lisDefault})
|
|
|
|
// Start a management server to act as the non-default authority.
|
|
nonDefaultAuthorityServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: lisNonDefault})
|
|
|
|
// Create a bootstrap configuration with two non-default authorities which
|
|
// have empty server configs, and therefore end up using the default server
|
|
// config, which points to the above management server.
|
|
nodeID := uuid.New().String()
|
|
|
|
resourceTypes := map[string]xdsclient.ResourceType{xdsresource.V3ListenerURL: listenerType}
|
|
ext := grpctransport.ServerIdentifierExtension{ConfigName: "insecure"}
|
|
siDefault := clients.ServerIdentifier{
|
|
ServerURI: defaultAuthorityServer.Address,
|
|
Extensions: ext,
|
|
}
|
|
siNonDefault := clients.ServerIdentifier{
|
|
ServerURI: nonDefaultAuthorityServer.Address,
|
|
Extensions: ext,
|
|
}
|
|
|
|
configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}}
|
|
xdsClientConfig := xdsclient.Config{
|
|
Servers: []xdsclient.ServerConfig{{ServerIdentifier: siDefault}},
|
|
Node: clients.Node{ID: nodeID},
|
|
TransportBuilder: grpctransport.NewBuilder(configs),
|
|
ResourceTypes: resourceTypes,
|
|
// Xdstp style resource names used in this test use a slash removed
|
|
// version of t.Name as their authority, and the empty config
|
|
// results in the top-level xds server configuration being used for
|
|
// this authority.
|
|
Authorities: map[string]xdsclient.Authority{
|
|
testAuthority1: {XDSServers: []xdsclient.ServerConfig{}},
|
|
testAuthority2: {XDSServers: []xdsclient.ServerConfig{}},
|
|
testAuthority3: {XDSServers: []xdsclient.ServerConfig{{ServerIdentifier: siNonDefault}}},
|
|
},
|
|
}
|
|
|
|
// Create an xDS client with the above config.
|
|
overrideWatchExpiryTimeout(t, defaultTestWatchExpiryTimeout)
|
|
client, err := xdsclient.New(xdsClientConfig)
|
|
if err != nil {
|
|
t.Fatalf("Failed to create xDS client: %v", err)
|
|
}
|
|
|
|
resources := e2e.UpdateOptions{
|
|
NodeID: nodeID,
|
|
Listeners: []*v3listenerpb.Listener{
|
|
e2e.DefaultClientListener(authorityTestResourceName11, rdsName),
|
|
e2e.DefaultClientListener(authorityTestResourceName12, rdsName),
|
|
e2e.DefaultClientListener(authorityTestResourceName2, rdsName),
|
|
e2e.DefaultClientListener(authorityTestResourceName3, rdsName),
|
|
},
|
|
SkipValidation: true,
|
|
}
|
|
if err := defaultAuthorityServer.Update(ctx, resources); err != nil {
|
|
t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err)
|
|
}
|
|
return lisDefault, lisNonDefault, client
|
|
}
|
|
|
|
// Tests the xdsChannel sharing logic among authorities. The test verifies the
|
|
// following scenarios:
|
|
// - A watch for a resource name with an authority matching an existing watch
|
|
// should not result in a new transport being created.
|
|
// - A watch for a resource name with different authority name but same
|
|
// authority config as an existing watch should not result in a new transport
|
|
// being created.
|
|
func (s) TestAuthority_XDSChannelSharing(t *testing.T) {
|
|
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
|
|
defer cancel()
|
|
lis, _, client := setupForAuthorityTests(ctx, t)
|
|
defer client.Close()
|
|
|
|
// Verify that no connection is established to the management server at this
|
|
// point. A transport is created only when a resource (which belongs to that
|
|
// authority) is requested.
|
|
sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
|
|
defer sCancel()
|
|
if _, err := lis.NewConnCh.Receive(sCtx); err != context.DeadlineExceeded {
|
|
t.Fatal("Unexpected new transport created to management server")
|
|
}
|
|
|
|
// Request the first resource. Verify that a new transport is created.
|
|
watcher := noopListenerWatcher{}
|
|
ldsCancel1 := client.WatchResource(xdsresource.V3ListenerURL, authorityTestResourceName11, watcher)
|
|
defer ldsCancel1()
|
|
if _, err := lis.NewConnCh.Receive(ctx); err != nil {
|
|
t.Fatalf("Timed out when waiting for a new transport to be created to the management server: %v", err)
|
|
}
|
|
|
|
// Request the second resource. Verify that no new transport is created.
|
|
ldsCancel2 := client.WatchResource(xdsresource.V3ListenerURL, authorityTestResourceName12, watcher)
|
|
defer ldsCancel2()
|
|
sCtx, sCancel = context.WithTimeout(ctx, defaultTestShortTimeout)
|
|
defer sCancel()
|
|
if _, err := lis.NewConnCh.Receive(sCtx); err != context.DeadlineExceeded {
|
|
t.Fatal("Unexpected new transport created to management server")
|
|
}
|
|
|
|
// Request the third resource. Verify that no new transport is created.
|
|
ldsCancel3 := client.WatchResource(xdsresource.V3ListenerURL, authorityTestResourceName2, watcher)
|
|
defer ldsCancel3()
|
|
sCtx, sCancel = context.WithTimeout(ctx, defaultTestShortTimeout)
|
|
defer sCancel()
|
|
if _, err := lis.NewConnCh.Receive(sCtx); err != context.DeadlineExceeded {
|
|
t.Fatal("Unexpected new transport created to management server")
|
|
}
|
|
}
|
|
|
|
// Test the xdsChannel close logic. The test verifies that the xDS client
|
|
// closes an xdsChannel immediately after the last watch is canceled.
|
|
func (s) TestAuthority_XDSChannelClose(t *testing.T) {
|
|
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
|
|
defer cancel()
|
|
lis, _, client := setupForAuthorityTests(ctx, t)
|
|
defer client.Close()
|
|
|
|
// Request the first resource. Verify that a new transport is created.
|
|
watcher := noopListenerWatcher{}
|
|
ldsCancel1 := client.WatchResource(xdsresource.V3ListenerURL, authorityTestResourceName11, watcher)
|
|
val, err := lis.NewConnCh.Receive(ctx)
|
|
if err != nil {
|
|
t.Fatalf("Timed out when waiting for a new transport to be created to the management server: %v", err)
|
|
}
|
|
conn := val.(*testutils.ConnWrapper)
|
|
|
|
// Request the second resource. Verify that no new transport is created.
|
|
ldsCancel2 := client.WatchResource(xdsresource.V3ListenerURL, authorityTestResourceName12, watcher)
|
|
sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
|
|
defer sCancel()
|
|
if _, err := lis.NewConnCh.Receive(sCtx); err != context.DeadlineExceeded {
|
|
t.Fatal("Unexpected new transport created to management server")
|
|
}
|
|
|
|
// Cancel both watches, and verify that the connection to the management
|
|
// server is closed.
|
|
ldsCancel1()
|
|
ldsCancel2()
|
|
if _, err := conn.CloseCh.Receive(ctx); err != nil {
|
|
t.Fatal("Timeout when waiting for connection to management server to be closed")
|
|
}
|
|
}
|
|
|
|
// Tests the scenario where the primary management server is unavailable at
|
|
// startup and the xDS client falls back to the secondary. The test verifies
|
|
// that the resource watcher is not notified of the connectivity failure until
|
|
// all servers have failed.
|
|
func (s) TestAuthority_Fallback(t *testing.T) {
|
|
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
|
|
defer cancel()
|
|
|
|
// Create primary and secondary management servers with restartable
|
|
// listeners.
|
|
l, err := net.Listen("tcp", "localhost:0")
|
|
if err != nil {
|
|
t.Fatalf("net.Listen() failed: %v", err)
|
|
}
|
|
primaryLis := testutils.NewRestartableListener(l)
|
|
primaryMgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: primaryLis})
|
|
l, err = net.Listen("tcp", "localhost:0")
|
|
if err != nil {
|
|
t.Fatalf("net.Listen() failed: %v", err)
|
|
}
|
|
secondaryLis := testutils.NewRestartableListener(l)
|
|
secondaryMgmtServer := e2e.StartManagementServer(t, e2e.ManagementServerOptions{Listener: secondaryLis})
|
|
|
|
nodeID := uuid.New().String()
|
|
|
|
resourceTypes := map[string]xdsclient.ResourceType{xdsresource.V3ListenerURL: listenerType}
|
|
psi := clients.ServerIdentifier{
|
|
ServerURI: primaryMgmtServer.Address,
|
|
Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"},
|
|
}
|
|
ssi := clients.ServerIdentifier{
|
|
ServerURI: secondaryMgmtServer.Address,
|
|
Extensions: grpctransport.ServerIdentifierExtension{ConfigName: "insecure"},
|
|
}
|
|
|
|
// Create config with the above primary and fallback management servers,
|
|
// and an xDS client with that configuration.
|
|
configs := map[string]grpctransport.Config{"insecure": {Credentials: insecure.NewBundle()}}
|
|
xdsClientConfig := xdsclient.Config{
|
|
Servers: []xdsclient.ServerConfig{{ServerIdentifier: psi}, {ServerIdentifier: ssi}},
|
|
Node: clients.Node{ID: nodeID},
|
|
TransportBuilder: grpctransport.NewBuilder(configs),
|
|
ResourceTypes: resourceTypes,
|
|
// Xdstp resource names used in this test do not specify an
|
|
// authority. These will end up looking up an entry with the
|
|
// empty key in the authorities map. Having an entry with an
|
|
// empty key and empty configuration, results in these
|
|
// resources also using the top-level configuration.
|
|
Authorities: map[string]xdsclient.Authority{
|
|
"": {XDSServers: []xdsclient.ServerConfig{}},
|
|
},
|
|
}
|
|
|
|
// Create an xDS client with the above config.
|
|
client, err := xdsclient.New(xdsClientConfig)
|
|
if err != nil {
|
|
t.Fatalf("Failed to create xDS client: %v", err)
|
|
}
|
|
defer client.Close()
|
|
|
|
const listenerName = "listener"
|
|
const rdsPrimaryName = "rds-primary"
|
|
const rdsSecondaryName = "rds-secondary"
|
|
|
|
// Create a Cluster resource on the primary.
|
|
resources := e2e.UpdateOptions{
|
|
NodeID: nodeID,
|
|
Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(listenerName, rdsPrimaryName)},
|
|
SkipValidation: true,
|
|
}
|
|
if err := primaryMgmtServer.Update(ctx, resources); err != nil {
|
|
t.Fatalf("Failed to update primary management server with resources: %v, err: %v", resources, err)
|
|
}
|
|
|
|
// Create a Cluster resource on the secondary .
|
|
resources = e2e.UpdateOptions{
|
|
NodeID: nodeID,
|
|
Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(listenerName, rdsSecondaryName)},
|
|
SkipValidation: true,
|
|
}
|
|
if err := secondaryMgmtServer.Update(ctx, resources); err != nil {
|
|
t.Fatalf("Failed to update primary management server with resources: %v, err: %v", resources, err)
|
|
}
|
|
|
|
// Stop the primary.
|
|
primaryLis.Close()
|
|
|
|
// Register a watch.
|
|
watcher := newListenerWatcher()
|
|
ldsCancel := client.WatchResource(xdsresource.V3ListenerURL, listenerName, watcher)
|
|
defer ldsCancel()
|
|
|
|
// Ensure that the connectivity error callback is not called. Since, this
|
|
// is the first watch without cached resource, it checks for resourceErrCh
|
|
sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout)
|
|
defer sCancel()
|
|
if v, err := watcher.resourceErrCh.Receive(sCtx); err != context.DeadlineExceeded {
|
|
t.Fatalf("Resource error callback on the watcher with error: %v", v.(error))
|
|
}
|
|
|
|
// Ensure that the resource update callback is invoked.
|
|
wantUpdate := listenerUpdateErrTuple{
|
|
update: listenerUpdate{
|
|
RouteConfigName: rdsSecondaryName,
|
|
},
|
|
}
|
|
if err := verifyListenerUpdate(ctx, watcher.updateCh, wantUpdate); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
// Stop the secondary.
|
|
secondaryLis.Close()
|
|
|
|
// Ensure that the connectivity error callback is called as ambient error
|
|
// since cached resource exist.
|
|
if _, err := watcher.ambientErrCh.Receive(ctx); err != nil {
|
|
t.Fatal("Timeout when waiting for ambient error callback on the watcher")
|
|
}
|
|
}
|