Refacter API server emphasis on logging

* To aid in debugging log API request and response bodies at trace
  level. Events can be correlated using the X-Reference-Id.
* Server now echos X-Reference-Id from client if set, otherwise
  generates an unique id.
* Move logic for X-Reference-Id into middleware
* Change uses of Header.Add() to Set() when setting Content-Type
* Log API operations in Apache format using gorilla middleware
* Port server code to use BaseContext and ConnContext

Fixes #10053

Signed-off-by: Jhon Honce <jhonce@redhat.com>

pkg/api/server/handler_panic.go

@@ -0,0 +1,32 @@
+package server
+
+import (
+	"fmt"
+	"net/http"
+	"runtime"
+
+	"github.com/containers/podman/v3/pkg/api/handlers/utils"
+	"github.com/gorilla/mux"
+	"github.com/sirupsen/logrus"
+)
+
+// panicHandler captures panics from endpoint handlers and logs stack trace
+func panicHandler() mux.MiddlewareFunc {
+	return func(h http.Handler) http.Handler {
+		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			// http.Server hides panics from handlers, we want to record them and fix the cause
+			defer func() {
+				err := recover()
+				if err != nil {
+					buf := make([]byte, 1<<20)
+					n := runtime.Stack(buf, true)
+					logrus.Warnf("Recovering from API service endpoint handler panic: %v, %s", err, buf[:n])
+					// Try to inform client things went south... won't work if handler already started writing response body
+					utils.InternalServerError(w, fmt.Errorf("%v", err))
+				}
+			}()
+
+			h.ServeHTTP(w, r)
+		})
+	}
+}