diff --git a/balancer/xds/edsbalancer/edsbalancer.go b/balancer/xds/edsbalancer/edsbalancer.go
index 483228fe..80b2f027 100644
--- a/balancer/xds/edsbalancer/edsbalancer.go
+++ b/balancer/xds/edsbalancer/edsbalancer.go
@@ -277,7 +277,9 @@ func (xdsB *EDSBalancer) UpdateBalancerState(s connectivity.State, p balancer.Pi
 
 // Close closes the balancer.
 func (xdsB *EDSBalancer) Close() {
-	xdsB.bg.close()
+	if xdsB.bg != nil {
+		xdsB.bg.close()
+	}
 }
 
 type dropPicker struct {
diff --git a/balancer/xds/edsbalancer/edsbalancer_test.go b/balancer/xds/edsbalancer/edsbalancer_test.go
index b1d7c339..5bdddd1e 100644
--- a/balancer/xds/edsbalancer/edsbalancer_test.go
+++ b/balancer/xds/edsbalancer/edsbalancer_test.go
@@ -337,6 +337,13 @@ func TestEDS_TwoLocalities(t *testing.T) {
 	}
 }
 
+func TestClose(t *testing.T) {
+	edsb := NewXDSBalancer(nil, nil)
+	// This is what could happen when switching between fallback and eds. This
+	// make sure it doesn't panic.
+	edsb.Close()
+}
+
 func init() {
 	balancer.Register(&testConstBalancerBuilder{})
 }