diff --git a/backoff.go b/backoff.go new file mode 100644 index 00000000..d0113ec9 --- /dev/null +++ b/backoff.go @@ -0,0 +1,68 @@ +package grpc + +import ( + "math/rand" + "time" +) + +// DefaultBackoffConfig uses values specified for backoff in +// https://github.com/grpc/grpc/blob/master/doc/connection-backoff.md. +var ( + DefaultBackoffConfig = &BackoffConfig{ + MaxDelay: 120 * time.Second, + baseDelay: 1.0 * time.Second, + factor: 1.6, + jitter: 0.2, + } +) + +// backoffStrategy defines the methodology for backing off after a grpc +// connection failure. +// +// This is unexported until the GRPC project decides whether or not to allow +// alternative backoff strategies. Once a decision is made, this type and its +// method may be exported. +type backoffStrategy interface { + // backoff returns the amount of time to wait before the next retry given + // the number of consecutive failures. + backoff(retries int) time.Duration +} + +// BackoffConfig defines the parameters for the default GRPC backoff strategy. +type BackoffConfig struct { + // MaxDelay is the upper bound of backoff delay. + MaxDelay time.Duration + + // TODO(stevvooe): The following fields are not exported, as allowing changes + + // baseDelay is the amount of time to wait before retrying after the first + // failure. + baseDelay time.Duration + + // factor is applied to the backoff after each retry. + factor float64 + + // jitter provides a range to randomize backoff delays. + jitter float64 +} + +func (bc *BackoffConfig) backoff(retries int) (t time.Duration) { + if retries == 0 { + return bc.baseDelay + } + backoff, max := float64(bc.baseDelay), float64(bc.MaxDelay) + for backoff < max && retries > 0 { + backoff *= bc.factor + retries-- + } + if backoff > max { + backoff = max + } + // Randomize backoff delays so that if a cluster of requests start at + // the same time, they won't operate in lockstep. + backoff *= 1 + bc.jitter*(rand.Float64()*2-1) + if backoff < 0 { + return 0 + } + return time.Duration(backoff) +} diff --git a/clientconn.go b/clientconn.go index e2264236..0f58d50f 100644 --- a/clientconn.go +++ b/clientconn.go @@ -75,6 +75,7 @@ type dialOptions struct { codec Codec cp Compressor dc Decompressor + bs backoffStrategy picker Picker block bool insecure bool @@ -114,6 +115,22 @@ func WithPicker(p Picker) DialOption { } } +// WithBackoffConfig configures the dialer to use the provided backoff +// parameters after connection failures. +func WithBackoffConfig(b *BackoffConfig) DialOption { + return withBackoff(b) +} + +// withBackoff sets the backoff strategy used for retries after a +// failed connection attempt. +// +// This can be exported if arbitrary backoff strategies are allowed by GRPC. +func withBackoff(bs backoffStrategy) DialOption { + return func(o *dialOptions) { + o.bs = bs + } +} + // WithBlock returns a DialOption which makes caller of Dial blocks until the underlying // connection is up. Without this, Dial returns immediately and connecting the server // happens in background. @@ -180,6 +197,11 @@ func Dial(target string, opts ...DialOption) (*ClientConn, error) { // Set the default codec. cc.dopts.codec = protoCodec{} } + + if cc.dopts.bs == nil { + cc.dopts.bs = DefaultBackoffConfig + } + if cc.dopts.picker == nil { cc.dopts.picker = &unicastPicker{ target: target, @@ -416,7 +438,7 @@ func (cc *Conn) resetTransport(closeTransport bool) error { return ErrClientConnTimeout } } - sleepTime := backoff(retries) + sleepTime := cc.dopts.bs.backoff(retries) timeout := sleepTime if timeout < minConnectTimeout { timeout = minConnectTimeout diff --git a/rpc_util.go b/rpc_util.go index 96c790be..adbafeaa 100644 --- a/rpc_util.go +++ b/rpc_util.go @@ -41,9 +41,7 @@ import ( "io" "io/ioutil" "math" - "math/rand" "os" - "time" "github.com/golang/protobuf/proto" "golang.org/x/net/context" @@ -411,38 +409,6 @@ func convertCode(err error) codes.Code { return codes.Unknown } -const ( - // how long to wait after the first failure before retrying - baseDelay = 1.0 * time.Second - // upper bound of backoff delay - maxDelay = 120 * time.Second - // backoff increases by this factor on each retry - backoffFactor = 1.6 - // backoff is randomized downwards by this factor - backoffJitter = 0.2 -) - -func backoff(retries int) (t time.Duration) { - if retries == 0 { - return baseDelay - } - backoff, max := float64(baseDelay), float64(maxDelay) - for backoff < max && retries > 0 { - backoff *= backoffFactor - retries-- - } - if backoff > max { - backoff = max - } - // Randomize backoff delays so that if a cluster of requests start at - // the same time, they won't operate in lockstep. - backoff *= 1 + backoffJitter*(rand.Float64()*2-1) - if backoff < 0 { - return 0 - } - return time.Duration(backoff) -} - // SupportPackageIsVersion1 is referenced from generated protocol buffer files // to assert that that code is compatible with this version of the grpc package. //