dns resolver: exponential retry when getting empty address list (#2201)

resolver/dns/dns_resolver.go

@@ -33,6 +33,7 @@ import (
 
 	"golang.org/x/net/context"
 	"google.golang.org/grpc/grpclog"
+	"google.golang.org/grpc/internal/backoff"
 	"google.golang.org/grpc/internal/grpcrand"
 	"google.golang.org/grpc/resolver"
 )
@@ -62,12 +63,12 @@ var (
 
 // NewBuilder creates a dnsBuilder which is used to factory DNS resolvers.
 func NewBuilder() resolver.Builder {
-	return &dnsBuilder{freq: defaultFreq}
+	return &dnsBuilder{minFreq: defaultFreq}
 }
 
 type dnsBuilder struct {
-	// frequency of polling the DNS server.
+	// minimum frequency of polling the DNS server.
-	freq time.Duration
+	minFreq time.Duration
 }
 
 // Build creates and starts a DNS resolver that watches the name resolution of the target.
@@ -98,7 +99,8 @@ func (b *dnsBuilder) Build(target resolver.Target, cc resolver.ClientConn, opts
 	// DNS address (non-IP).
 	ctx, cancel := context.WithCancel(context.Background())
 	d := &dnsResolver{
-		freq:                 b.freq,
+		freq:                 b.minFreq,
+		backoff:              backoff.Exponential{MaxDelay: b.minFreq},
 		host:                 host,
 		port:                 port,
 		ctx:                  ctx,
@@ -155,6 +157,8 @@ func (i *ipResolver) watcher() {
 // dnsResolver watches for the name resolution update for a non-IP target.
 type dnsResolver struct {
 	freq       time.Duration
+	backoff    backoff.Exponential
+	retryCount int
 	host       string
 	port       string
 	ctx        context.Context
@@ -198,8 +202,15 @@ func (d *dnsResolver) watcher() {
 		case <-d.rn:
 		}
 		result, sc := d.lookup()
-		// Next lookup should happen after an interval defined by d.freq.
+		// Next lookup should happen within an interval defined by d.freq. It may be
+		// more often due to exponential retry on empty address list.
+		if len(result) == 0 {
+			d.retryCount++
+			d.t.Reset(d.backoff.Backoff(d.retryCount))
+		} else {
+			d.retryCount = 0
 			d.t.Reset(d.freq)
+		}
 		d.cc.NewServiceConfig(sc)
 		d.cc.NewAddress(result)
 	}

resolver/dns/dns_resolver_test.go

@@ -46,7 +46,7 @@ type testClientConn struct {
 	target string
 	m1     sync.Mutex
 	addrs  []resolver.Address
-	a      int
+	a      int // how many times NewAddress() has been called
 	m2     sync.Mutex
 	sc     string
 	s      int
@@ -936,3 +936,60 @@ func TestDisableServiceConfig(t *testing.T) {
 		r.Close()
 	}
 }
+
+func TestDNSResolverRetry(t *testing.T) {
+	b := NewBuilder()
+	target := "ipv4.single.fake"
+	cc := &testClientConn{target: target}
+	r, err := b.Build(resolver.Target{Endpoint: target}, cc, resolver.BuildOption{})
+	if err != nil {
+		t.Fatalf("%v\n", err)
+	}
+	var addrs []resolver.Address
+	for {
+		addrs, _ = cc.getAddress()
+		if len(addrs) == 1 {
+			break
+		}
+		time.Sleep(time.Millisecond)
+	}
+	want := []resolver.Address{{Addr: "1.2.3.4" + colonDefaultPort}}
+	if !reflect.DeepEqual(want, addrs) {
+		t.Errorf("Resolved addresses of target: %q = %+v, want %+v\n", target, addrs, want)
+	}
+	// mutate the host lookup table so the target has 0 address returned.
+	revertTbl := mutateTbl(target)
+	// trigger a resolve that will get empty address list
+	r.ResolveNow(resolver.ResolveNowOption{})
+	for {
+		addrs, _ = cc.getAddress()
+		if len(addrs) == 0 {
+			break
+		}
+		time.Sleep(time.Millisecond)
+	}
+	revertTbl()
+	// wait for the retry to happen in two seconds.
+	timer := time.NewTimer(2 * time.Second)
+	for {
+		b := false
+		select {
+		case <-timer.C:
+			b = true
+		default:
+			addrs, _ = cc.getAddress()
+			if len(addrs) == 1 {
+				b = true
+				break
+			}
+			time.Sleep(time.Millisecond)
+		}
+		if b {
+			break
+		}
+	}
+	if !reflect.DeepEqual(want, addrs) {
+		t.Errorf("Resolved addresses of target: %q = %+v, want %+v\n", target, addrs, want)
+	}
+	r.Close()
+}