From 6d22a67a30bb9bd45468087956d866d7b6391f3c Mon Sep 17 00:00:00 2001 From: bergquist Date: Wed, 6 Sep 2017 22:24:10 +0200 Subject: [PATCH] return /metrics before session middleware --- pkg/api/api.go | 6 - pkg/api/http_server.go | 11 ++ pkg/metrics/graphitebridge/graphite.go | 12 ++ pkg/metrics/graphitebridge/graphite_test.go | 40 ++++++ pkg/metrics/metrics.go | 29 ++++- pkg/middleware/request_metrics.go | 135 ++++++++++++++++++++ 6 files changed, 223 insertions(+), 10 deletions(-) diff --git a/pkg/api/api.go b/pkg/api/api.go index b57b0ccefc9..72d2093b536 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -6,7 +6,6 @@ import ( "github.com/grafana/grafana/pkg/api/dtos" "github.com/grafana/grafana/pkg/middleware" m "github.com/grafana/grafana/pkg/models" - "github.com/prometheus/client_golang/prometheus/promhttp" ) // Register adds http routes @@ -98,8 +97,6 @@ func (hs *HttpServer) registerRoutes() { // api renew session based on remember cookie r.Get("/api/login/ping", quota("session"), LoginApiPing) - r.Get("/metrics", promhttp.Handler()) - // authed api r.Group("/api", func() { @@ -266,9 +263,6 @@ func (hs *HttpServer) registerRoutes() { r.Get("/tsdb/testdata/gensql", reqGrafanaAdmin, wrap(GenerateSqlTestData)) r.Get("/tsdb/testdata/random-walk", wrap(GetTestDataRandomWalk)) - // metrics - //r.Get("/metrics", wrap(GetInternalMetrics)) - r.Group("/alerts", func() { r.Post("/test", bind(dtos.AlertTestCommand{}), wrap(AlertTest)) r.Post("/:alertId/pause", bind(dtos.PauseAlertCommand{}), wrap(PauseAlert), reqEditorRole) diff --git a/pkg/api/http_server.go b/pkg/api/http_server.go index 54f08197bae..0f5e9ac8369 100644 --- a/pkg/api/http_server.go +++ b/pkg/api/http_server.go @@ -11,6 +11,8 @@ import ( "path" "time" + "github.com/prometheus/client_golang/prometheus/promhttp" + gocache "github.com/patrickmn/go-cache" macaron "gopkg.in/macaron.v1" @@ -165,6 +167,7 @@ func (hs *HttpServer) newMacaron() *macaron.Macaron { })) m.Use(hs.healthHandler) + m.Use(hs.metricsEndpoint) m.Use(middleware.GetContextHandler()) m.Use(middleware.Sessioner(&setting.SessionOptions)) m.Use(middleware.RequestMetrics()) @@ -180,6 +183,14 @@ func (hs *HttpServer) newMacaron() *macaron.Macaron { return m } +func (hs *HttpServer) metricsEndpoint(ctx *macaron.Context) { + if ctx.Req.Method != "GET" || ctx.Req.URL.Path != "/metrics" { + return + } + + promhttp.Handler().ServeHTTP(ctx.Resp, ctx.Req.Request) +} + func (hs *HttpServer) healthHandler(ctx *macaron.Context) { if ctx.Req.Method != "GET" || ctx.Req.URL.Path != "/api/health" { return diff --git a/pkg/metrics/graphitebridge/graphite.go b/pkg/metrics/graphitebridge/graphite.go index b9c766a03e1..7a115a06587 100644 --- a/pkg/metrics/graphitebridge/graphite.go +++ b/pkg/metrics/graphitebridge/graphite.go @@ -206,6 +206,18 @@ func (b *Bridge) writeMetrics(w io.Writer, mfs []*dto.MetricFamily, prefix strin return err } + ignoreThisMetric := false + for _, v := range ignorePrefix { + if strings.HasPrefix(mf.GetName(), v) { + ignoreThisMetric = true + break + } + } + + if ignoreThisMetric { + continue + } + buf := bufio.NewWriter(w) for _, s := range vec { if err := writeSanitized(buf, prefix); err != nil { diff --git a/pkg/metrics/graphitebridge/graphite_test.go b/pkg/metrics/graphitebridge/graphite_test.go index 1994fbde5a9..3f56fd8cea0 100644 --- a/pkg/metrics/graphitebridge/graphite_test.go +++ b/pkg/metrics/graphitebridge/graphite_test.go @@ -343,6 +343,46 @@ func TestCounter(t *testing.T) { } } +func TestCanIgnoreSomeMetrics(t *testing.T) { + cntVec := prometheus.NewCounter( + prometheus.CounterOpts{ + Name: "http_request_total", + Help: "docstring", + ConstLabels: prometheus.Labels{"constname": "constvalue"}, + }) + + reg := prometheus.NewRegistry() + reg.MustRegister(cntVec) + + cntVec.Inc() + + b, err := NewBridge(&Config{ + URL: "localhost:8080", + Gatherer: reg, + CountersAsDelta: true, + }) + if err != nil { + t.Fatalf("error creating bridge: %v", err) + } + + // first collect + mfs, err := reg.Gather() + if err != nil { + t.Fatalf("error: %v", err) + } + + var buf bytes.Buffer + err = b.writeMetrics(&buf, mfs, "prefix", model.Time(1477043083)) + if err != nil { + t.Fatalf("error: %v", err) + } + + want := "" + if got := buf.String(); want != got { + t.Fatalf("wanted \n%s\n, got \n%s\n", want, got) + } +} + func TestPush(t *testing.T) { reg := prometheus.NewRegistry() cntVec := prometheus.NewCounterVec( diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 0a398cea069..989639fd83f 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -16,10 +16,12 @@ import ( ) var ( - M_Instance_Start prometheus.Counter - M_Page_Status *prometheus.CounterVec - M_Api_Status *prometheus.CounterVec - M_Proxy_Status *prometheus.CounterVec + M_Instance_Start prometheus.Counter + M_Page_Status *prometheus.CounterVec + M_Api_Status *prometheus.CounterVec + M_Proxy_Status *prometheus.CounterVec + M_Http_Request_Total *prometheus.CounterVec + M_Http_Request_Summary *prometheus.SummaryVec M_Api_User_SignUpStarted prometheus.Counter M_Api_User_SignUpCompleted prometheus.Counter @@ -84,6 +86,22 @@ func init() { []string{"code"}, ) + M_Http_Request_Total = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "http_request_total", + Help: "http request counter", + }, + []string{"code", "method"}, + ) + + M_Http_Request_Summary = prometheus.NewSummaryVec( + prometheus.SummaryOpts{ + Name: "http_request_duration", + Help: "http request summary", + }, + []string{"code", "method"}, + ) + M_Api_User_SignUpStarted = prometheus.NewCounter(prometheus.CounterOpts{ Name: "api_user_signup_started_total", Help: "amount of users who started the signup flow", @@ -214,6 +232,9 @@ func initMetricVars(settings *MetricSettings) { M_Instance_Start, M_Page_Status, M_Api_Status, + M_Proxy_Status, + M_Http_Request_Total, + M_Http_Request_Summary, M_Api_User_SignUpStarted, M_Api_User_SignUpCompleted, M_Api_User_SignUpInvite, diff --git a/pkg/middleware/request_metrics.go b/pkg/middleware/request_metrics.go index 3fc3d55727d..6cda2e477d3 100644 --- a/pkg/middleware/request_metrics.go +++ b/pkg/middleware/request_metrics.go @@ -2,7 +2,9 @@ package middleware import ( "net/http" + "strconv" "strings" + "time" "github.com/grafana/grafana/pkg/metrics" "gopkg.in/macaron.v1" @@ -11,10 +13,16 @@ import ( func RequestMetrics() macaron.Handler { return func(res http.ResponseWriter, req *http.Request, c *macaron.Context) { rw := res.(macaron.ResponseWriter) + now := time.Now() c.Next() status := rw.Status() + code := sanitizeCode(status) + method := sanitizeMethod(req.Method) + metrics.M_Http_Request_Total.WithLabelValues(code, method).Inc() + metrics.M_Http_Request_Summary.WithLabelValues(code, method).Observe(time.Since(now).Seconds()) + if strings.HasPrefix(req.RequestURI, "/api/datasources/proxy") { countProxyRequests(status) } else if strings.HasPrefix(req.RequestURI, "/api/") { @@ -63,3 +71,130 @@ func countProxyRequests(status int) { metrics.M_Proxy_Status.WithLabelValues("unknown").Inc() } } + +func sanitizeMethod(m string) string { + switch m { + case "GET", "get": + return "get" + case "PUT", "put": + return "put" + case "HEAD", "head": + return "head" + case "POST", "post": + return "post" + case "DELETE", "delete": + return "delete" + case "CONNECT", "connect": + return "connect" + case "OPTIONS", "options": + return "options" + case "NOTIFY", "notify": + return "notify" + default: + return strings.ToLower(m) + } +} + +// If the wrapped http.Handler has not set a status code, i.e. the value is +// currently 0, santizeCode will return 200, for consistency with behavior in +// the stdlib. +func sanitizeCode(s int) string { + switch s { + case 100: + return "100" + case 101: + return "101" + + case 200, 0: + return "200" + case 201: + return "201" + case 202: + return "202" + case 203: + return "203" + case 204: + return "204" + case 205: + return "205" + case 206: + return "206" + + case 300: + return "300" + case 301: + return "301" + case 302: + return "302" + case 304: + return "304" + case 305: + return "305" + case 307: + return "307" + + case 400: + return "400" + case 401: + return "401" + case 402: + return "402" + case 403: + return "403" + case 404: + return "404" + case 405: + return "405" + case 406: + return "406" + case 407: + return "407" + case 408: + return "408" + case 409: + return "409" + case 410: + return "410" + case 411: + return "411" + case 412: + return "412" + case 413: + return "413" + case 414: + return "414" + case 415: + return "415" + case 416: + return "416" + case 417: + return "417" + case 418: + return "418" + + case 500: + return "500" + case 501: + return "501" + case 502: + return "502" + case 503: + return "503" + case 504: + return "504" + case 505: + return "505" + + case 428: + return "428" + case 429: + return "429" + case 431: + return "431" + case 511: + return "511" + + default: + return strconv.Itoa(s) + } +}