Also call storage on mode1. Measure latency (#87739)

* Also call storage on mode1. Add metrics

* Update comment

* Don't use compare function for now

* Remove very important space

* Finish add logging in mode2.
Also call US in mode1 in a non blocking way

* Improve code readability on modes 1 and 2

* Fix tests

* Rename vars

* Lint

* Return error from legacy write

* Renume useless defer

* [REVIEW] improvements

* Pass kind instead of name

* Use kind instead of name in metrics

* Only call latency metrics once

* Return error on writes to legacystore in mode1

* Move accesssor logic into the goroutine as well
This commit is contained in:
Leonor Oliveira
2024-05-22 09:23:29 +01:00
committed by GitHub
parent 7c5c62f617
commit dd771e818e
5 changed files with 244 additions and 70 deletions

View File

@ -1,6 +1,11 @@
package rest
import "github.com/prometheus/client_golang/prometheus"
import (
"strconv"
"time"
"github.com/prometheus/client_golang/prometheus"
)
type dualWriterMetrics struct {
legacy *prometheus.HistogramVec
@ -14,7 +19,7 @@ var DualWriterStorageDuration = prometheus.NewHistogramVec(prometheus.HistogramO
Help: "Histogram for the runtime of dual writer storage duration per mode",
Namespace: "grafana",
NativeHistogramBucketFactor: 1.1,
}, []string{"status_code", "mode", "name", "method"})
}, []string{"is_error", "mode", "kind", "method"})
// DualWriterLegacyDuration is a metric summary for dual writer legacy duration per mode
var DualWriterLegacyDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
@ -22,7 +27,7 @@ var DualWriterLegacyDuration = prometheus.NewHistogramVec(prometheus.HistogramOp
Help: "Histogram for the runtime of dual writer legacy duration per mode",
Namespace: "grafana",
NativeHistogramBucketFactor: 1.1,
}, []string{"status_code", "mode", "name", "method"})
}, []string{"is_error", "mode", "kind", "method"})
// DualWriterOutcome is a metric summary for dual writer outcome comparison between the 2 stores per mode
var DualWriterOutcome = prometheus.NewHistogramVec(prometheus.HistogramOpts{
@ -30,7 +35,7 @@ var DualWriterOutcome = prometheus.NewHistogramVec(prometheus.HistogramOpts{
Help: "Histogram for the runtime of dual writer outcome comparison between the 2 stores per mode",
Namespace: "grafana",
NativeHistogramBucketFactor: 1.1,
}, []string{"mode", "name", "outcome", "method"})
}, []string{"mode", "name", "method"})
func (m *dualWriterMetrics) init() {
m.legacy = DualWriterLegacyDuration
@ -38,17 +43,21 @@ func (m *dualWriterMetrics) init() {
m.outcome = DualWriterOutcome
}
// nolint:unused
func (m *dualWriterMetrics) recordLegacyDuration(statusCode string, mode string, name string, method string, duration float64) {
m.legacy.WithLabelValues(statusCode, mode, name, method).Observe(duration)
func (m *dualWriterMetrics) recordLegacyDuration(isError bool, mode string, name string, method string, startFrom time.Time) {
duration := time.Since(startFrom).Seconds()
m.legacy.WithLabelValues(strconv.FormatBool(isError), mode, name, method).Observe(duration)
}
func (m *dualWriterMetrics) recordStorageDuration(isError bool, mode string, name string, method string, startFrom time.Time) {
duration := time.Since(startFrom).Seconds()
m.storage.WithLabelValues(strconv.FormatBool(isError), mode, name, method).Observe(duration)
}
// nolint:unused
func (m *dualWriterMetrics) recordStorageDuration(statusCode string, mode string, name string, method string, duration float64) {
m.storage.WithLabelValues(statusCode, mode, name, method).Observe(duration)
}
// nolint:unused
func (m *dualWriterMetrics) recordOutcome(mode string, name string, outcome string, method string) {
m.outcome.WithLabelValues(mode, name, outcome, method).Observe(1)
func (m *dualWriterMetrics) recordOutcome(mode string, name string, outcome bool, method string) {
var observeValue float64
if outcome {
observeValue = 1
}
m.outcome.WithLabelValues(mode, name, method).Observe(observeValue)
}