diff --git a/server.go b/server.go
index 927b1c5e..bdf68a0f 100644
--- a/server.go
+++ b/server.go
@@ -248,7 +248,6 @@ func (s *Server) Serve(lis net.Listener) error {
 		delete(s.lis, lis)
 		s.mu.Unlock()
 	}()
-	listenerAddr := lis.Addr()
 	for {
 		rawConn, err := lis.Accept()
 		if err != nil {
@@ -259,13 +258,13 @@ func (s *Server) Serve(lis net.Listener) error {
 		}
 		// Start a new goroutine to deal with rawConn
 		// so we don't stall this Accept loop goroutine.
-		go s.handleRawConn(listenerAddr, rawConn)
+		go s.handleRawConn(rawConn)
 	}
 }
 
 // handleRawConn is run in its own goroutine and handles a just-accepted
 // connection that has not had any I/O performed on it yet.
-func (s *Server) handleRawConn(listenerAddr net.Addr, rawConn net.Conn) {
+func (s *Server) handleRawConn(rawConn net.Conn) {
 	conn, authInfo, err := s.useTransportAuthenticator(rawConn)
 	if err != nil {
 		s.mu.Lock()
@@ -285,7 +284,7 @@ func (s *Server) handleRawConn(listenerAddr net.Addr, rawConn net.Conn) {
 	s.mu.Unlock()
 
 	if s.opts.useHandlerImpl {
-		s.serveUsingHandler(listenerAddr, conn)
+		s.serveUsingHandler(conn)
 	} else {
 		s.serveNewHTTP2Transport(conn, authInfo)
 	}
@@ -341,29 +340,18 @@ var _ http.Handler = (*Server)(nil)
 // method as one of the environment types.
 //
 // conn is the *tls.Conn that's already been authenticated.
-func (s *Server) serveUsingHandler(listenerAddr net.Addr, conn net.Conn) {
+func (s *Server) serveUsingHandler(conn net.Conn) {
 	if !s.addConn(conn) {
 		conn.Close()
 		return
 	}
 	defer s.removeConn(conn)
-	connDone := make(chan struct{})
-	hs := &http.Server{
-		Handler: s,
-		ConnState: func(c net.Conn, cs http.ConnState) {
-			if cs == http.StateClosed {
-				close(connDone)
-			}
-		},
-	}
-	if err := http2.ConfigureServer(hs, &http2.Server{
+	h2s := &http2.Server{
 		MaxConcurrentStreams: s.opts.maxConcurrentStreams,
-	}); err != nil {
-		grpclog.Fatalf("grpc: http2.ConfigureServer: %v", err)
-		return
 	}
-	hs.Serve(&singleConnListener{addr: listenerAddr, conn: conn})
-	<-connDone
+	h2s.ServeConn(conn, &http2.ServeConnOpts{
+		Handler: s,
+	})
 }
 
 func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
@@ -756,30 +744,3 @@ func SetTrailer(ctx context.Context, md metadata.MD) error {
 	}
 	return stream.SetTrailer(md)
 }
-
-// singleConnListener is a net.Listener that yields a single conn.
-type singleConnListener struct {
-	mu   sync.Mutex
-	addr net.Addr
-	conn net.Conn // nil if done
-}
-
-func (ln *singleConnListener) Addr() net.Addr { return ln.addr }
-
-func (ln *singleConnListener) Close() error {
-	ln.mu.Lock()
-	defer ln.mu.Unlock()
-	ln.conn = nil
-	return nil
-}
-
-func (ln *singleConnListener) Accept() (net.Conn, error) {
-	ln.mu.Lock()
-	defer ln.mu.Unlock()
-	c := ln.conn
-	if c == nil {
-		return nil, io.EOF
-	}
-	ln.conn = nil
-	return c, nil
-}