From b1807c31892b7450d5b5735be087159dbb587223 Mon Sep 17 00:00:00 2001 From: lyuxuan Date: Tue, 10 Apr 2018 14:27:42 -0700 Subject: [PATCH] Channelz: more stable tesing (#1983) --- test/channelz_test.go | 224 ++++++++++++++++++++++++++---------------- 1 file changed, 142 insertions(+), 82 deletions(-) diff --git a/test/channelz_test.go b/test/channelz_test.go index 572bf5a09..0492e3138 100644 --- a/test/channelz_test.go +++ b/test/channelz_test.go @@ -19,6 +19,7 @@ package test import ( + "fmt" "net" "testing" "time" @@ -46,6 +47,18 @@ func (te *test) startServers(ts testpb.TestServiceServer, num int) { } } +func verifyResultWithDelay(f func() (bool, error)) error { + var ok bool + var err error + for i := 0; i < 1000; i++ { + if ok, err = f(); ok { + return nil + } + time.Sleep(10 * time.Millisecond) + } + return err +} + func TestCZServerRegistrationAndDeletion(t *testing.T) { defer leakcheck.Check(t) testcases := []struct { @@ -104,19 +117,27 @@ func TestCZTopChannelRegistrationAndDeletion(t *testing.T) { te.srvAddr = "" ccs = append(ccs, cc) } - time.Sleep(10 * time.Millisecond) - tcs, end := channelz.GetTopChannels(c.start) - if len(tcs) != c.length || end != c.end { - t.Fatalf("GetTopChannels(%d) = %+v (len of which: %d), end: %+v, want len(GetTopChannels(%d)) = %d, end: %+v", c.start, tcs, len(tcs), end, c.start, c.length, c.end) + if err := verifyResultWithDelay(func() (bool, error) { + if tcs, end := channelz.GetTopChannels(c.start); len(tcs) != c.length || end != c.end { + return false, fmt.Errorf("GetTopChannels(%d) = %+v (len of which: %d), end: %+v, want len(GetTopChannels(%d)) = %d, end: %+v", c.start, tcs, len(tcs), end, c.start, c.length, c.end) + } + return true, nil + }); err != nil { + t.Fatal(err) } + for _, cc := range ccs { cc.Close() } - tcs, end = channelz.GetTopChannels(c.start) - if len(tcs) != 0 || !end { - t.Fatalf("GetTopChannels(0) = %+v (len of which: %d), end: %+v, want len(GetTopChannels(0)) = 0, end: true", tcs, len(tcs), end) - } + if err := verifyResultWithDelay(func() (bool, error) { + if tcs, end := channelz.GetTopChannels(c.start); len(tcs) != 0 || !end { + return false, fmt.Errorf("GetTopChannels(0) = %+v (len of which: %d), end: %+v, want len(GetTopChannels(0)) = 0, end: true", tcs, len(tcs), end) + } + return true, nil + }); err != nil { + t.Fatal(err) + } te.tearDown() } } @@ -135,28 +156,36 @@ func TestCZNestedChannelRegistrationAndDeletion(t *testing.T) { te.resolverScheme = r.Scheme() te.clientConn() defer te.tearDown() - time.Sleep(10 * time.Millisecond) - tcs, _ := channelz.GetTopChannels(0) - if len(tcs) != 1 { - t.Fatalf("There should only be one top channel, not %d", len(tcs)) - } - if len(tcs[0].NestedChans) != 1 { - t.Fatalf("There should be one nested channel from grpclb, not %d", len(tcs[0].NestedChans)) + + if err := verifyResultWithDelay(func() (bool, error) { + tcs, _ := channelz.GetTopChannels(0) + if len(tcs) != 1 { + return false, fmt.Errorf("There should only be one top channel, not %d", len(tcs)) + } + if len(tcs[0].NestedChans) != 1 { + return false, fmt.Errorf("There should be one nested channel from grpclb, not %d", len(tcs[0].NestedChans)) + } + return true, nil + }); err != nil { + t.Fatal(err) } r.NewServiceConfig(`{"loadBalancingPolicy": "round_robin"}`) r.NewAddress([]resolver.Address{{Addr: "127.0.0.1:0"}}) // wait for the shutdown of grpclb balancer - time.Sleep(10 * time.Millisecond) - tcs, _ = channelz.GetTopChannels(0) - if len(tcs) != 1 { - t.Fatalf("There should only be one top channel, not %d", len(tcs)) + if err := verifyResultWithDelay(func() (bool, error) { + tcs, _ := channelz.GetTopChannels(0) + if len(tcs) != 1 { + return false, fmt.Errorf("There should only be one top channel, not %d", len(tcs)) + } + if len(tcs[0].NestedChans) != 0 { + return false, fmt.Errorf("There should be 0 nested channel from grpclb, not %d", len(tcs[0].NestedChans)) + } + return true, nil + }); err != nil { + t.Fatal(err) } - if len(tcs[0].NestedChans) != 0 { - t.Fatalf("There should be 0 nested channel from grpclb, not %d", len(tcs[0].NestedChans)) - } - } func TestCZClientSubChannelSocketRegistrationAndDeletion(t *testing.T) { @@ -178,44 +207,57 @@ func TestCZClientSubChannelSocketRegistrationAndDeletion(t *testing.T) { defer te.tearDown() // Here, we just wait for all sockets to be up. In the future, if we implement // IDLE, we may need to make several rpc calls to create the sockets. - time.Sleep(100 * time.Millisecond) - tcs, _ := channelz.GetTopChannels(0) - if len(tcs) != 1 { - t.Fatalf("There should only be one top channel, not %d", len(tcs)) - } - if len(tcs[0].SubChans) != num { - t.Fatalf("There should be %d subchannel not %d", num, len(tcs[0].SubChans)) - } - count := 0 - for k := range tcs[0].SubChans { - sc := channelz.GetSubChannel(k) - if sc == nil { - t.Fatalf("got subchannel") + if err := verifyResultWithDelay(func() (bool, error) { + tcs, _ := channelz.GetTopChannels(0) + if len(tcs) != 1 { + return false, fmt.Errorf("There should only be one top channel, not %d", len(tcs)) } - count += len(sc.Sockets) - } - if count != num { - t.Fatalf("There should be %d sockets not %d", num, count) + if len(tcs[0].SubChans) != num { + return false, fmt.Errorf("There should be %d subchannel not %d", num, len(tcs[0].SubChans)) + } + count := 0 + for k := range tcs[0].SubChans { + sc := channelz.GetSubChannel(k) + if sc == nil { + return false, fmt.Errorf("got subchannel") + } + count += len(sc.Sockets) + } + if count != num { + return false, fmt.Errorf("There should be %d sockets not %d", num, count) + } + + return true, nil + }); err != nil { + t.Fatal(err) } r.NewAddress(svrAddrs[:len(svrAddrs)-1]) - time.Sleep(100 * time.Millisecond) - tcs, _ = channelz.GetTopChannels(0) - if len(tcs[0].SubChans) != num-1 { - t.Fatalf("There should be %d subchannel not %d", num-1, len(tcs[0].SubChans)) - } - count = 0 - for k := range tcs[0].SubChans { - sc := channelz.GetSubChannel(k) - if sc == nil { - t.Fatalf("got subchannel") - } - count += len(sc.Sockets) - } - if count != num-1 { - t.Fatalf("There should be %d sockets not %d", num-1, count) - } + if err := verifyResultWithDelay(func() (bool, error) { + tcs, _ := channelz.GetTopChannels(0) + if len(tcs) != 1 { + return false, fmt.Errorf("There should only be one top channel, not %d", len(tcs)) + } + if len(tcs[0].SubChans) != num-1 { + return false, fmt.Errorf("There should be %d subchannel not %d", num-1, len(tcs[0].SubChans)) + } + count := 0 + for k := range tcs[0].SubChans { + sc := channelz.GetSubChannel(k) + if sc == nil { + return false, fmt.Errorf("got subchannel") + } + count += len(sc.Sockets) + } + if count != num-1 { + return false, fmt.Errorf("There should be %d sockets not %d", num-1, count) + } + + return true, nil + }); err != nil { + t.Fatal(err) + } } func TestCZServerSocketRegistrationAndDeletion(t *testing.T) { @@ -237,25 +279,35 @@ func TestCZServerSocketRegistrationAndDeletion(t *testing.T) { c.Close() } }() - time.Sleep(10 * time.Millisecond) - ss, _ := channelz.GetServers(0) - if len(ss) != 1 { - t.Fatalf("There should only be one server, not %d", len(ss)) - } - if len(ss[0].ListenSockets) != 1 { - t.Fatalf("There should only be one server listen socket, not %d", len(ss[0].ListenSockets)) - } - ns, _ := channelz.GetServerSockets(ss[0].ID, 0) - if len(ns) != num { - t.Fatalf("There should be %d normal sockets not %d", num, len(ns)) + var svrID int64 + if err := verifyResultWithDelay(func() (bool, error) { + ss, _ := channelz.GetServers(0) + if len(ss) != 1 { + return false, fmt.Errorf("There should only be one server, not %d", len(ss)) + } + if len(ss[0].ListenSockets) != 1 { + return false, fmt.Errorf("There should only be one server listen socket, not %d", len(ss[0].ListenSockets)) + } + ns, _ := channelz.GetServerSockets(ss[0].ID, 0) + if len(ns) != num { + return false, fmt.Errorf("There should be %d normal sockets not %d", num, len(ns)) + } + svrID = ss[0].ID + return true, nil + }); err != nil { + t.Fatal(err) } ccs[len(ccs)-1].Close() - time.Sleep(10 * time.Millisecond) - ns, _ = channelz.GetServerSockets(ss[0].ID, 0) - if len(ns) != num-1 { - t.Fatalf("There should be %d normal sockets not %d", num-1, len(ns)) + if err := verifyResultWithDelay(func() (bool, error) { + ns, _ := channelz.GetServerSockets(svrID, 0) + if len(ns) != num-1 { + return false, fmt.Errorf("There should be %d normal sockets not %d", num-1, len(ns)) + } + return true, nil + }); err != nil { + t.Fatal(err) } } @@ -268,20 +320,28 @@ func TestCZServerListenSocketDeletion(t *testing.T) { t.Fatalf("failed to listen: %v", err) } go s.Serve(lis) - time.Sleep(10 * time.Millisecond) - ss, _ := channelz.GetServers(0) - if len(ss) != 1 { - t.Fatalf("There should only be one server, not %d", len(ss)) - } - if len(ss[0].ListenSockets) != 1 { - t.Fatalf("There should only be one server listen socket, not %d", len(ss[0].ListenSockets)) + if err := verifyResultWithDelay(func() (bool, error) { + ss, _ := channelz.GetServers(0) + if len(ss) != 1 { + return false, fmt.Errorf("There should only be one server, not %d", len(ss)) + } + if len(ss[0].ListenSockets) != 1 { + return false, fmt.Errorf("There should only be one server listen socket, not %d", len(ss[0].ListenSockets)) + } + return true, nil + }); err != nil { + t.Fatal(err) } lis.Close() - time.Sleep(10 * time.Millisecond) - ss, _ = channelz.GetServers(0) - if len(ss) != 1 { - t.Fatalf("There should be 1 server, not %d", len(ss)) + if err := verifyResultWithDelay(func() (bool, error) { + ss, _ := channelz.GetServers(0) + if len(ss) != 1 { + return false, fmt.Errorf("There should be 1 server, not %d", len(ss)) + } + return true, nil + }); err != nil { + t.Fatal(err) } s.Stop() }