transport/server: flush GOAWAY before closing conn due to max age (#5821)

Fixes https://github.com/grpc/grpc-go/issues/4859

internal/transport/controlbuf.go

@@ -209,6 +209,14 @@ type outFlowControlSizeRequest struct {
 
 func (*outFlowControlSizeRequest) isTransportResponseFrame() bool { return false }
 
+// closeConnection is an instruction to tell the loopy writer to flush the
+// framer and exit, which will cause the transport's connection to be closed
+// (by the client or server).  The transport itself will close after the reader
+// encounters the EOF caused by the connection closure.
+type closeConnection struct{}
+
+func (closeConnection) isTransportResponseFrame() bool { return false }
+
 type outStreamState int
 
 const (
@@ -817,6 +825,14 @@ func (l *loopyWriter) goAwayHandler(g *goAway) error {
 	return nil
 }
 
+func (l *loopyWriter) closeConnectionHandler() error {
+	l.framer.writer.Flush()
+	// Exit loopyWriter entirely by returning an error here.  This will lead to
+	// the transport closing the connection, and, ultimately, transport
+	// closure.
+	return ErrConnClosing
+}
+
 func (l *loopyWriter) handle(i interface{}) error {
 	switch i := i.(type) {
 	case *incomingWindowUpdate:
@@ -845,6 +861,8 @@ func (l *loopyWriter) handle(i interface{}) error {
 		return l.goAwayHandler(i)
 	case *outFlowControlSizeRequest:
 		return l.outFlowControlSizeRequestHandler(i)
+	case closeConnection:
+		return l.closeConnectionHandler()
 	default:
 		return fmt.Errorf("transport: unknown control message type %T", i)
 	}

internal/transport/http2_server.go

@@ -1153,7 +1153,7 @@ func (t *http2Server) keepalive() {
 				if logger.V(logLevel) {
 					logger.Infof("transport: closing server transport due to maximum connection age.")
 				}
-				t.Close()
+				t.controlBuf.put(closeConnection{})
 			case <-t.done:
 			}
 			return

test/goaway_test.go

@@ -97,7 +97,7 @@ func (s) TestDetailedGoAwayErrorOnGracefulClosePropagatesToRPCError(t *testing.T
 	sopts := []grpc.ServerOption{
 		grpc.KeepaliveParams(keepalive.ServerParameters{
 			MaxConnectionAge:      time.Millisecond * 100,
-			MaxConnectionAgeGrace: time.Millisecond,
+			MaxConnectionAgeGrace: time.Nanosecond, // ~instantaneously, but non-zero to avoid default
 		}),
 	}
 	if err := ss.Start(sopts); err != nil {