control: add retrial logic in 'Stop'

This commit is contained in:
Gyu-Ho Lee 2017-01-24 12:02:09 -08:00
parent 6a6290d64a
commit aef15ae7a0
No known key found for this signature in database
GPG Key ID: 1DDD39C7EB70C24C
3 changed files with 12 additions and 3 deletions

View File

@ -99,7 +99,7 @@ func commandFunc(cmd *cobra.Command, args []string) error {
println()
time.Sleep(5 * time.Second)
if err := step3StopDatabase(cfg); err != nil {
return err
plog.Warning(err)
}
println()

View File

@ -19,7 +19,6 @@ import (
"time"
"github.com/coreos/dbtester/agent/agentpb"
"google.golang.org/grpc"
)

View File

@ -16,6 +16,7 @@ package control
import (
"fmt"
"time"
"github.com/coreos/dbtester/agent/agentpb"
)
@ -24,7 +25,16 @@ func step3StopDatabase(cfg Config) error {
switch cfg.Step3.Action {
case "stop":
plog.Info("step 3: stopping databases...")
return bcastReq(cfg, agentpb.Request_Stop)
var err error
for i := 0; i < 5; i++ {
if err = bcastReq(cfg, agentpb.Request_Stop); err != nil {
plog.Warningf("STOP failed at %s", cfg.PeerIPs[i])
time.Sleep(300 * time.Millisecond)
continue
}
break
}
return err
default:
return fmt.Errorf("unknown %q", cfg.Step3.Action)