Channelz: more stable tesing (#1983)

This commit is contained in:
lyuxuan 2018-04-10 14:27:42 -07:00 committed by GitHub
parent 844b2a520b
commit b1807c3189
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 142 additions and 82 deletions

View File

@ -19,6 +19,7 @@
package test package test
import ( import (
"fmt"
"net" "net"
"testing" "testing"
"time" "time"
@ -46,6 +47,18 @@ func (te *test) startServers(ts testpb.TestServiceServer, num int) {
} }
} }
func verifyResultWithDelay(f func() (bool, error)) error {
var ok bool
var err error
for i := 0; i < 1000; i++ {
if ok, err = f(); ok {
return nil
}
time.Sleep(10 * time.Millisecond)
}
return err
}
func TestCZServerRegistrationAndDeletion(t *testing.T) { func TestCZServerRegistrationAndDeletion(t *testing.T) {
defer leakcheck.Check(t) defer leakcheck.Check(t)
testcases := []struct { testcases := []struct {
@ -104,19 +117,27 @@ func TestCZTopChannelRegistrationAndDeletion(t *testing.T) {
te.srvAddr = "" te.srvAddr = ""
ccs = append(ccs, cc) ccs = append(ccs, cc)
} }
time.Sleep(10 * time.Millisecond) if err := verifyResultWithDelay(func() (bool, error) {
tcs, end := channelz.GetTopChannels(c.start) if tcs, end := channelz.GetTopChannels(c.start); len(tcs) != c.length || end != c.end {
if len(tcs) != c.length || end != c.end { return false, fmt.Errorf("GetTopChannels(%d) = %+v (len of which: %d), end: %+v, want len(GetTopChannels(%d)) = %d, end: %+v", c.start, tcs, len(tcs), end, c.start, c.length, c.end)
t.Fatalf("GetTopChannels(%d) = %+v (len of which: %d), end: %+v, want len(GetTopChannels(%d)) = %d, end: %+v", c.start, tcs, len(tcs), end, c.start, c.length, c.end) }
return true, nil
}); err != nil {
t.Fatal(err)
} }
for _, cc := range ccs { for _, cc := range ccs {
cc.Close() cc.Close()
} }
tcs, end = channelz.GetTopChannels(c.start)
if len(tcs) != 0 || !end {
t.Fatalf("GetTopChannels(0) = %+v (len of which: %d), end: %+v, want len(GetTopChannels(0)) = 0, end: true", tcs, len(tcs), end)
}
if err := verifyResultWithDelay(func() (bool, error) {
if tcs, end := channelz.GetTopChannels(c.start); len(tcs) != 0 || !end {
return false, fmt.Errorf("GetTopChannels(0) = %+v (len of which: %d), end: %+v, want len(GetTopChannels(0)) = 0, end: true", tcs, len(tcs), end)
}
return true, nil
}); err != nil {
t.Fatal(err)
}
te.tearDown() te.tearDown()
} }
} }
@ -135,28 +156,36 @@ func TestCZNestedChannelRegistrationAndDeletion(t *testing.T) {
te.resolverScheme = r.Scheme() te.resolverScheme = r.Scheme()
te.clientConn() te.clientConn()
defer te.tearDown() defer te.tearDown()
time.Sleep(10 * time.Millisecond)
tcs, _ := channelz.GetTopChannels(0) if err := verifyResultWithDelay(func() (bool, error) {
if len(tcs) != 1 { tcs, _ := channelz.GetTopChannels(0)
t.Fatalf("There should only be one top channel, not %d", len(tcs)) if len(tcs) != 1 {
} return false, fmt.Errorf("There should only be one top channel, not %d", len(tcs))
if len(tcs[0].NestedChans) != 1 { }
t.Fatalf("There should be one nested channel from grpclb, not %d", len(tcs[0].NestedChans)) if len(tcs[0].NestedChans) != 1 {
return false, fmt.Errorf("There should be one nested channel from grpclb, not %d", len(tcs[0].NestedChans))
}
return true, nil
}); err != nil {
t.Fatal(err)
} }
r.NewServiceConfig(`{"loadBalancingPolicy": "round_robin"}`) r.NewServiceConfig(`{"loadBalancingPolicy": "round_robin"}`)
r.NewAddress([]resolver.Address{{Addr: "127.0.0.1:0"}}) r.NewAddress([]resolver.Address{{Addr: "127.0.0.1:0"}})
// wait for the shutdown of grpclb balancer // wait for the shutdown of grpclb balancer
time.Sleep(10 * time.Millisecond) if err := verifyResultWithDelay(func() (bool, error) {
tcs, _ = channelz.GetTopChannels(0) tcs, _ := channelz.GetTopChannels(0)
if len(tcs) != 1 { if len(tcs) != 1 {
t.Fatalf("There should only be one top channel, not %d", len(tcs)) return false, fmt.Errorf("There should only be one top channel, not %d", len(tcs))
}
if len(tcs[0].NestedChans) != 0 {
return false, fmt.Errorf("There should be 0 nested channel from grpclb, not %d", len(tcs[0].NestedChans))
}
return true, nil
}); err != nil {
t.Fatal(err)
} }
if len(tcs[0].NestedChans) != 0 {
t.Fatalf("There should be 0 nested channel from grpclb, not %d", len(tcs[0].NestedChans))
}
} }
func TestCZClientSubChannelSocketRegistrationAndDeletion(t *testing.T) { func TestCZClientSubChannelSocketRegistrationAndDeletion(t *testing.T) {
@ -178,44 +207,57 @@ func TestCZClientSubChannelSocketRegistrationAndDeletion(t *testing.T) {
defer te.tearDown() defer te.tearDown()
// Here, we just wait for all sockets to be up. In the future, if we implement // Here, we just wait for all sockets to be up. In the future, if we implement
// IDLE, we may need to make several rpc calls to create the sockets. // IDLE, we may need to make several rpc calls to create the sockets.
time.Sleep(100 * time.Millisecond) if err := verifyResultWithDelay(func() (bool, error) {
tcs, _ := channelz.GetTopChannels(0) tcs, _ := channelz.GetTopChannels(0)
if len(tcs) != 1 { if len(tcs) != 1 {
t.Fatalf("There should only be one top channel, not %d", len(tcs)) return false, fmt.Errorf("There should only be one top channel, not %d", len(tcs))
}
if len(tcs[0].SubChans) != num {
t.Fatalf("There should be %d subchannel not %d", num, len(tcs[0].SubChans))
}
count := 0
for k := range tcs[0].SubChans {
sc := channelz.GetSubChannel(k)
if sc == nil {
t.Fatalf("got <nil> subchannel")
} }
count += len(sc.Sockets) if len(tcs[0].SubChans) != num {
} return false, fmt.Errorf("There should be %d subchannel not %d", num, len(tcs[0].SubChans))
if count != num { }
t.Fatalf("There should be %d sockets not %d", num, count) count := 0
for k := range tcs[0].SubChans {
sc := channelz.GetSubChannel(k)
if sc == nil {
return false, fmt.Errorf("got <nil> subchannel")
}
count += len(sc.Sockets)
}
if count != num {
return false, fmt.Errorf("There should be %d sockets not %d", num, count)
}
return true, nil
}); err != nil {
t.Fatal(err)
} }
r.NewAddress(svrAddrs[:len(svrAddrs)-1]) r.NewAddress(svrAddrs[:len(svrAddrs)-1])
time.Sleep(100 * time.Millisecond)
tcs, _ = channelz.GetTopChannels(0)
if len(tcs[0].SubChans) != num-1 {
t.Fatalf("There should be %d subchannel not %d", num-1, len(tcs[0].SubChans))
}
count = 0
for k := range tcs[0].SubChans {
sc := channelz.GetSubChannel(k)
if sc == nil {
t.Fatalf("got <nil> subchannel")
}
count += len(sc.Sockets)
}
if count != num-1 {
t.Fatalf("There should be %d sockets not %d", num-1, count)
}
if err := verifyResultWithDelay(func() (bool, error) {
tcs, _ := channelz.GetTopChannels(0)
if len(tcs) != 1 {
return false, fmt.Errorf("There should only be one top channel, not %d", len(tcs))
}
if len(tcs[0].SubChans) != num-1 {
return false, fmt.Errorf("There should be %d subchannel not %d", num-1, len(tcs[0].SubChans))
}
count := 0
for k := range tcs[0].SubChans {
sc := channelz.GetSubChannel(k)
if sc == nil {
return false, fmt.Errorf("got <nil> subchannel")
}
count += len(sc.Sockets)
}
if count != num-1 {
return false, fmt.Errorf("There should be %d sockets not %d", num-1, count)
}
return true, nil
}); err != nil {
t.Fatal(err)
}
} }
func TestCZServerSocketRegistrationAndDeletion(t *testing.T) { func TestCZServerSocketRegistrationAndDeletion(t *testing.T) {
@ -237,25 +279,35 @@ func TestCZServerSocketRegistrationAndDeletion(t *testing.T) {
c.Close() c.Close()
} }
}() }()
time.Sleep(10 * time.Millisecond) var svrID int64
ss, _ := channelz.GetServers(0) if err := verifyResultWithDelay(func() (bool, error) {
if len(ss) != 1 { ss, _ := channelz.GetServers(0)
t.Fatalf("There should only be one server, not %d", len(ss)) if len(ss) != 1 {
} return false, fmt.Errorf("There should only be one server, not %d", len(ss))
if len(ss[0].ListenSockets) != 1 { }
t.Fatalf("There should only be one server listen socket, not %d", len(ss[0].ListenSockets)) if len(ss[0].ListenSockets) != 1 {
} return false, fmt.Errorf("There should only be one server listen socket, not %d", len(ss[0].ListenSockets))
ns, _ := channelz.GetServerSockets(ss[0].ID, 0) }
if len(ns) != num { ns, _ := channelz.GetServerSockets(ss[0].ID, 0)
t.Fatalf("There should be %d normal sockets not %d", num, len(ns)) if len(ns) != num {
return false, fmt.Errorf("There should be %d normal sockets not %d", num, len(ns))
}
svrID = ss[0].ID
return true, nil
}); err != nil {
t.Fatal(err)
} }
ccs[len(ccs)-1].Close() ccs[len(ccs)-1].Close()
time.Sleep(10 * time.Millisecond)
ns, _ = channelz.GetServerSockets(ss[0].ID, 0) if err := verifyResultWithDelay(func() (bool, error) {
if len(ns) != num-1 { ns, _ := channelz.GetServerSockets(svrID, 0)
t.Fatalf("There should be %d normal sockets not %d", num-1, len(ns)) if len(ns) != num-1 {
return false, fmt.Errorf("There should be %d normal sockets not %d", num-1, len(ns))
}
return true, nil
}); err != nil {
t.Fatal(err)
} }
} }
@ -268,20 +320,28 @@ func TestCZServerListenSocketDeletion(t *testing.T) {
t.Fatalf("failed to listen: %v", err) t.Fatalf("failed to listen: %v", err)
} }
go s.Serve(lis) go s.Serve(lis)
time.Sleep(10 * time.Millisecond) if err := verifyResultWithDelay(func() (bool, error) {
ss, _ := channelz.GetServers(0) ss, _ := channelz.GetServers(0)
if len(ss) != 1 { if len(ss) != 1 {
t.Fatalf("There should only be one server, not %d", len(ss)) return false, fmt.Errorf("There should only be one server, not %d", len(ss))
} }
if len(ss[0].ListenSockets) != 1 { if len(ss[0].ListenSockets) != 1 {
t.Fatalf("There should only be one server listen socket, not %d", len(ss[0].ListenSockets)) return false, fmt.Errorf("There should only be one server listen socket, not %d", len(ss[0].ListenSockets))
}
return true, nil
}); err != nil {
t.Fatal(err)
} }
lis.Close() lis.Close()
time.Sleep(10 * time.Millisecond) if err := verifyResultWithDelay(func() (bool, error) {
ss, _ = channelz.GetServers(0) ss, _ := channelz.GetServers(0)
if len(ss) != 1 { if len(ss) != 1 {
t.Fatalf("There should be 1 server, not %d", len(ss)) return false, fmt.Errorf("There should be 1 server, not %d", len(ss))
}
return true, nil
}); err != nil {
t.Fatal(err)
} }
s.Stop() s.Stop()
} }