diff --git a/p2p/net/swarm/dial_test.go b/p2p/net/swarm/dial_test.go
index 6c3c4f920..d0e0cdbb0 100644
--- a/p2p/net/swarm/dial_test.go
+++ b/p2p/net/swarm/dial_test.go
@@ -420,18 +420,18 @@ func TestDialBackoffClears(t *testing.T) {
 	}
 	s1.peers.AddAddrs(s2.local, ifaceAddrs1, peer.PermanentAddrTTL)
 
-	before = time.Now()
+	if _, err := s1.Dial(ctx, s2.local); err == nil {
+		t.Fatal("should have failed to dial backed off peer")
+	}
+
+	time.Sleep(baseBackoffTime)
+
 	if c, err := s1.Dial(ctx, s2.local); err != nil {
 		t.Fatal(err)
 	} else {
 		c.Close()
 		t.Log("correctly connected")
 	}
-	duration = time.Now().Sub(before)
-
-	if duration >= dt {
-		// t.Error("took too long", duration, dt)
-	}
 
 	if s1.backf.Backoff(s2.local) {
 		t.Error("s2 should no longer be on backoff")
diff --git a/p2p/net/swarm/swarm_dial.go b/p2p/net/swarm/swarm_dial.go
index a2c135126..0bcdda1a6 100644
--- a/p2p/net/swarm/swarm_dial.go
+++ b/p2p/net/swarm/swarm_dial.go
@@ -147,44 +147,71 @@ func (ds *dialsync) Unlock(dst peer.ID) {
 //  	dialbackoff.Clear(p)
 //  }
 //
+
 type dialbackoff struct {
-	entries map[peer.ID]struct{}
+	entries map[peer.ID]*backoffPeer
 	lock    sync.RWMutex
 }
 
+type backoffPeer struct {
+	tries int
+	until time.Time
+}
+
 func (db *dialbackoff) init() {
 	if db.entries == nil {
-		db.entries = make(map[peer.ID]struct{})
+		db.entries = make(map[peer.ID]*backoffPeer)
 	}
 }
 
 // Backoff returns whether the client should backoff from dialing
-// peeer p
-func (db *dialbackoff) Backoff(p peer.ID) bool {
+// peer p
+func (db *dialbackoff) Backoff(p peer.ID) (backoff bool) {
 	db.lock.Lock()
+	defer db.lock.Unlock()
 	db.init()
-	_, found := db.entries[p]
-	db.lock.Unlock()
-	return found
+	bp, found := db.entries[p]
+	if found && time.Now().Before(bp.until) {
+		return true
+	}
+
+	return false
 }
 
+const baseBackoffTime = time.Second * 5
+const maxBackoffTime = time.Minute * 5
+
 // AddBackoff lets other nodes know that we've entered backoff with
 // peer p, so dialers should not wait unnecessarily. We still will
 // attempt to dial with one goroutine, in case we get through.
 func (db *dialbackoff) AddBackoff(p peer.ID) {
 	db.lock.Lock()
+	defer db.lock.Unlock()
 	db.init()
-	db.entries[p] = struct{}{}
-	db.lock.Unlock()
+	bp, ok := db.entries[p]
+	if !ok {
+		db.entries[p] = &backoffPeer{
+			tries: 1,
+			until: time.Now().Add(baseBackoffTime),
+		}
+		return
+	}
+
+	expTimeAdd := time.Second * time.Duration(bp.tries*bp.tries)
+	if expTimeAdd > maxBackoffTime {
+		expTimeAdd = maxBackoffTime
+	}
+	bp.until = time.Now().Add(baseBackoffTime + expTimeAdd)
+	bp.tries++
 }
 
 // Clear removes a backoff record. Clients should call this after a
 // successful Dial.
 func (db *dialbackoff) Clear(p peer.ID) {
 	db.lock.Lock()
+	defer db.lock.Unlock()
 	db.init()
 	delete(db.entries, p)
-	db.lock.Unlock()
 }
 
 // Dial connects to a peer.
diff --git a/p2p/net/swarm/swarm_listen.go b/p2p/net/swarm/swarm_listen.go
index d1bcb0752..b32e75a18 100644
--- a/p2p/net/swarm/swarm_listen.go
+++ b/p2p/net/swarm/swarm_listen.go
@@ -138,5 +138,8 @@ func (s *Swarm) connHandler(c *ps.Conn) *Conn {
 		return nil
 	}
 
+	// if a peer dials us, remove from dial backoff.
+	s.backf.Clear(sc.RemotePeer())
+
 	return sc
 }