Merge pull request #601 from stevvooe/backoff-strategy

backoff: allow configuration of backoff strategy

backoff.go

@@ -0,0 +1,68 @@
+package grpc
+
+import (
+	"math/rand"
+	"time"
+)
+
+// DefaultBackoffConfig uses values specified for backoff in
+// https://github.com/grpc/grpc/blob/master/doc/connection-backoff.md.
+var (
+	DefaultBackoffConfig = &BackoffConfig{
+		MaxDelay:  120 * time.Second,
+		baseDelay: 1.0 * time.Second,
+		factor:    1.6,
+		jitter:    0.2,
+	}
+)
+
+// backoffStrategy defines the methodology for backing off after a grpc
+// connection failure.
+//
+// This is unexported until the GRPC project decides whether or not to allow
+// alternative backoff strategies. Once a decision is made, this type and its
+// method may be exported.
+type backoffStrategy interface {
+	// backoff returns the amount of time to wait before the next retry given
+	// the number of consecutive failures.
+	backoff(retries int) time.Duration
+}
+
+// BackoffConfig defines the parameters for the default GRPC backoff strategy.
+type BackoffConfig struct {
+	// MaxDelay is the upper bound of backoff delay.
+	MaxDelay time.Duration
+
+	// TODO(stevvooe): The following fields are not exported, as allowing changes
+
+	// baseDelay is the amount of time to wait before retrying after the first
+	// failure.
+	baseDelay time.Duration
+
+	// factor is applied to the backoff after each retry.
+	factor float64
+
+	// jitter provides a range to randomize backoff delays.
+	jitter float64
+}
+
+func (bc *BackoffConfig) backoff(retries int) (t time.Duration) {
+	if retries == 0 {
+		return bc.baseDelay
+	}
+	backoff, max := float64(bc.baseDelay), float64(bc.MaxDelay)
+	for backoff < max && retries > 0 {
+		backoff *= bc.factor
+		retries--
+	}
+	if backoff > max {
+		backoff = max
+	}
+	// Randomize backoff delays so that if a cluster of requests start at
+	// the same time, they won't operate in lockstep.
+	backoff *= 1 + bc.jitter*(rand.Float64()*2-1)
+	if backoff < 0 {
+		return 0
+	}
+	return time.Duration(backoff)
+}

clientconn.go

@@ -75,6 +75,7 @@ type dialOptions struct {
 	codec    Codec
 	cp       Compressor
 	dc       Decompressor
+	bs       backoffStrategy
 	picker   Picker
 	block    bool
 	insecure bool
@@ -114,6 +115,22 @@ func WithPicker(p Picker) DialOption {
 	}
 }
 
+// WithBackoffConfig configures the dialer to use the provided backoff
+// parameters after connection failures.
+func WithBackoffConfig(b *BackoffConfig) DialOption {
+	return withBackoff(b)
+}
+
+// withBackoff sets the backoff strategy used for retries after a
+// failed connection attempt.
+//
+// This can be exported if arbitrary backoff strategies are allowed by GRPC.
+func withBackoff(bs backoffStrategy) DialOption {
+	return func(o *dialOptions) {
+		o.bs = bs
+	}
+}
+
 // WithBlock returns a DialOption which makes caller of Dial blocks until the underlying
 // connection is up. Without this, Dial returns immediately and connecting the server
 // happens in background.
@@ -180,6 +197,11 @@ func Dial(target string, opts ...DialOption) (*ClientConn, error) {
 		// Set the default codec.
 		cc.dopts.codec = protoCodec{}
 	}
+
+	if cc.dopts.bs == nil {
+		cc.dopts.bs = DefaultBackoffConfig
+	}
+
 	if cc.dopts.picker == nil {
 		cc.dopts.picker = &unicastPicker{
 			target: target,
@@ -415,7 +437,7 @@ func (cc *Conn) resetTransport(closeTransport bool) error {
 				return ErrClientConnTimeout
 			}
 		}
-		sleepTime := backoff(retries)
+		sleepTime := cc.dopts.bs.backoff(retries)
 		timeout := sleepTime
 		if timeout < minConnectTimeout {
 			timeout = minConnectTimeout

rpc_util.go

@@ -41,9 +41,7 @@ import (
 	"io"
 	"io/ioutil"
 	"math"
-	"math/rand"
 	"os"
-	"time"
 
 	"github.com/golang/protobuf/proto"
 	"golang.org/x/net/context"
@@ -411,38 +409,6 @@ func convertCode(err error) codes.Code {
 	return codes.Unknown
 }
 
-const (
-	// how long to wait after the first failure before retrying
-	baseDelay = 1.0 * time.Second
-	// upper bound of backoff delay
-	maxDelay = 120 * time.Second
-	// backoff increases by this factor on each retry
-	backoffFactor = 1.6
-	// backoff is randomized downwards by this factor
-	backoffJitter = 0.2
-)
-
-func backoff(retries int) (t time.Duration) {
-	if retries == 0 {
-		return baseDelay
-	}
-	backoff, max := float64(baseDelay), float64(maxDelay)
-	for backoff < max && retries > 0 {
-		backoff *= backoffFactor
-		retries--
-	}
-	if backoff > max {
-		backoff = max
-	}
-	// Randomize backoff delays so that if a cluster of requests start at
-	// the same time, they won't operate in lockstep.
-	backoff *= 1 + backoffJitter*(rand.Float64()*2-1)
-	if backoff < 0 {
-		return 0
-	}
-	return time.Duration(backoff)
-}
-
 // SupportPackageIsVersion1 is referenced from generated protocol buffer files
 // to assert that that code is compatible with this version of the grpc package.
 //