refactor(stringlabels): Support stringlabels in loghttp, pattern and ruler tests. (#17102)

This commit is contained in:
Karsten Jeschkies
2025-04-11 07:41:52 +02:00
committed by GitHub
parent c7ffeb5211
commit 75593e0ed7
10 changed files with 73 additions and 122 deletions

View File

@@ -32,34 +32,27 @@ func TestEntryMarshalUnmarshalJSON(t *testing.T) {
{
name: "entry with structured metadata",
entry: Entry{
Timestamp: time.Unix(0, 123456789012345),
Line: "test line",
StructuredMetadata: labels.Labels{
{Name: "foo", Value: "bar"},
{Name: "count", Value: "42"},
},
Timestamp: time.Unix(0, 123456789012345),
Line: "test line",
StructuredMetadata: labels.FromStrings("foo", "bar", "count", "42"),
},
expected: `["123456789012345","test line",{"count":"42","foo":"bar"}]`,
},
{
name: "entry with newlines in structured metadata",
entry: Entry{
Timestamp: time.Unix(0, 123456789012345),
Line: "test line",
StructuredMetadata: labels.Labels{
{Name: "message", Value: "a\nb\nc"},
},
Timestamp: time.Unix(0, 123456789012345),
Line: "test line",
StructuredMetadata: labels.FromStrings("message", "a\nb\nc"),
},
expected: `["123456789012345","test line",{"message":"a\nb\nc"}]`,
},
{
name: "entry with quotes in structured metadata",
entry: Entry{
Timestamp: time.Unix(0, 123456789012345),
Line: "test line",
StructuredMetadata: labels.Labels{
{Name: "message", Value: `"test"`},
},
Timestamp: time.Unix(0, 123456789012345),
Line: "test line",
StructuredMetadata: labels.FromStrings("message", `"test"`),
},
expected: `["123456789012345","test line",{"message":"\"test\""}]`,
},
@@ -90,11 +83,9 @@ func TestEntryMarshalUnmarshalJSON(t *testing.T) {
func TestEntryRoundTripWithNewlines(t *testing.T) {
// This test specifically checks if newlines in structured metadata are preserved correctly
original := Entry{
Timestamp: time.Unix(0, 123456789012345),
Line: "test line",
StructuredMetadata: labels.Labels{
{Name: "message", Value: "a\nb\nc"},
},
Timestamp: time.Unix(0, 123456789012345),
Line: "test line",
StructuredMetadata: labels.FromStrings("message", "a\nb\nc"),
}
// First marshal to JSON
@@ -114,17 +105,15 @@ func TestEntryRoundTripWithNewlines(t *testing.T) {
require.NoError(t, err)
// Verify the structured metadata value still has actual newlines
require.Equal(t, "a\nb\nc", decoded.StructuredMetadata[0].Value)
require.Equal(t, "a\nb\nc", decoded.StructuredMetadata.Get("message"))
}
func TestEntryJsoniterEncoding(t *testing.T) {
// This test specifically checks if newlines in structured metadata are properly encoded using jsoniter
entry := Entry{
Timestamp: time.Unix(0, 123456789012345),
Line: "test line",
StructuredMetadata: labels.Labels{
{Name: "message", Value: "a\nb\nc"},
},
Timestamp: time.Unix(0, 123456789012345),
Line: "test line",
StructuredMetadata: labels.FromStrings("message", "a\nb\nc"),
}
// Use the custom jsoniter instance that has our extension registered
@@ -147,17 +136,15 @@ func TestEntryJsoniterEncoding(t *testing.T) {
// Verify the values match, especially the newlines in structured metadata
require.Equal(t, entry.Timestamp, decoded.Timestamp)
require.Equal(t, entry.Line, decoded.Line)
require.Equal(t, "a\nb\nc", decoded.StructuredMetadata[0].Value)
require.Equal(t, "a\nb\nc", decoded.StructuredMetadata.Get("message"))
}
func TestEntryEncoderSpecifically(t *testing.T) {
// Test the specific EntryEncoder.Encode method directly
entry := Entry{
Timestamp: time.Unix(0, 123456789012345),
Line: "test line",
StructuredMetadata: labels.Labels{
{Name: "message", Value: "a\nb\nc"},
},
Timestamp: time.Unix(0, 123456789012345),
Line: "test line",
StructuredMetadata: labels.FromStrings("message", "a\nb\nc"),
}
// Create a jsoniter stream
@@ -200,11 +187,11 @@ func TestUnmarshalEscapingIssue(t *testing.T) {
require.NoError(t, err)
// Print the actual value of StructuredMetadata for debugging
t.Logf("Structured metadata value after unmarshal: %#v", entry.StructuredMetadata[0].Value)
t.Logf("Structured metadata value after unmarshal: %#v", entry.StructuredMetadata.Get("message"))
// Verify the value still has actual newlines - this is the core issue
// We expect "a\nb\nc" when printed to log but in memory it should be a real newline
actualValue := entry.StructuredMetadata[0].Value
actualValue := entry.StructuredMetadata.Get("message")
require.Equal(t, "a\nb\nc", actualValue, "Newlines should be properly preserved")
// Also directly check the byte representation of the string to see what's happening
@@ -224,7 +211,7 @@ func TestUnmarshalEscapingIssue(t *testing.T) {
require.NoError(t, err)
// Check the value after double marshaling/unmarshaling
t.Logf("Value after second unmarshal: %#v", reUnmarshaled.StructuredMetadata[0].Value)
t.Logf("Value after second unmarshal: %#v", reUnmarshaled.StructuredMetadata.Get("message"))
}
func TestParseStringWithNewlines(t *testing.T) {

View File

@@ -154,21 +154,14 @@ func TestStreams_ToProto(t *testing.T) {
Labels: map[string]string{"job": "fake"},
Entries: []Entry{
{Timestamp: time.Unix(0, 1), Line: "1"},
{Timestamp: time.Unix(0, 2), Line: "2", StructuredMetadata: labels.Labels{
{Name: "foo", Value: "a"},
{Name: "bar", Value: "b"},
}},
{Timestamp: time.Unix(0, 2), Line: "2", StructuredMetadata: labels.FromStrings("foo", "a", "bar", "b")},
},
},
{
Labels: map[string]string{"job": "fake", "lvl": "error"},
Entries: []Entry{
{Timestamp: time.Unix(0, 3), Line: "3"},
{Timestamp: time.Unix(0, 4), Line: "4",
StructuredMetadata: labels.Labels{
{Name: "foo", Value: "a"},
{Name: "bar", Value: "b"},
}},
{Timestamp: time.Unix(0, 4), Line: "4", StructuredMetadata: labels.FromStrings("foo", "a", "bar", "b")},
},
},
},
@@ -178,8 +171,8 @@ func TestStreams_ToProto(t *testing.T) {
Entries: []logproto.Entry{
{Timestamp: time.Unix(0, 1), Line: "1"},
{Timestamp: time.Unix(0, 2), Line: "2", StructuredMetadata: []logproto.LabelAdapter{
{Name: "foo", Value: "a"},
{Name: "bar", Value: "b"},
{Name: "foo", Value: "a"},
}},
},
},
@@ -188,8 +181,8 @@ func TestStreams_ToProto(t *testing.T) {
Entries: []logproto.Entry{
{Timestamp: time.Unix(0, 3), Line: "3"},
{Timestamp: time.Unix(0, 4), Line: "4", StructuredMetadata: []logproto.LabelAdapter{
{Name: "foo", Value: "a"},
{Name: "bar", Value: "b"},
{Name: "foo", Value: "a"},
}},
},
},
@@ -224,10 +217,7 @@ func Test_QueryResponseUnmarshal(t *testing.T) {
Labels: LabelSet{"foo": "bar"},
Entries: []Entry{
{Timestamp: time.Unix(0, 1), Line: "1"},
{Timestamp: time.Unix(0, 2), Line: "2", StructuredMetadata: labels.Labels{
{Name: "foo", Value: "a"},
{Name: "bar", Value: "b"},
}},
{Timestamp: time.Unix(0, 2), Line: "2", StructuredMetadata: labels.FromStrings("foo", "a", "bar", "b")},
},
},
},
@@ -247,10 +237,7 @@ func Test_QueryResponseUnmarshal(t *testing.T) {
Labels: LabelSet{"foo": "bar"},
Entries: []Entry{
{Timestamp: time.Unix(0, 1), Line: "log line 1"},
{Timestamp: time.Unix(0, 2), Line: "some log line 2", StructuredMetadata: labels.Labels{
{Name: "foo", Value: "a"},
{Name: "bar", Value: "b"},
}},
{Timestamp: time.Unix(0, 2), Line: "some log line 2", StructuredMetadata: labels.FromStrings("foo", "a", "bar", "b")},
},
},
Stream{
@@ -260,10 +247,7 @@ func Test_QueryResponseUnmarshal(t *testing.T) {
{Timestamp: time.Unix(0, 2), Line: "2"},
{Timestamp: time.Unix(0, 2), Line: "2"},
{Timestamp: time.Unix(0, 2), Line: "2"},
{Timestamp: time.Unix(0, 2), Line: "2", StructuredMetadata: labels.Labels{
{Name: "foo", Value: "a"},
{Name: "bar", Value: "b"},
}},
{Timestamp: time.Unix(0, 2), Line: "2", StructuredMetadata: labels.FromStrings("foo", "a", "bar", "b")},
},
},
},

View File

@@ -334,13 +334,13 @@ func labelSet(keyVals ...string) labels.Labels {
panic("not matching key-value pairs")
}
lbls := labels.Labels{}
lbls := labels.NewBuilder(labels.EmptyLabels())
for i := 0; i < len(keyVals)-1; i += 2 {
lbls = append(lbls, labels.Label{Name: keyVals[i], Value: keyVals[i+1]})
lbls.Set(keyVals[i], keyVals[i+1])
}
return lbls
return lbls.Labels()
}
func testPayload() (time.Time, string) {

View File

@@ -319,9 +319,7 @@ func TestBuildNotifierConfig(t *testing.T) {
AlertManagerConfig: ruler_config.AlertManagerConfig{
AlertmanagerURL: "http://alertmanager.default.svc.cluster.local/alertmanager",
},
ExternalLabels: []labels.Label{
{Name: "region", Value: "us-east-1"},
},
ExternalLabels: labels.FromStrings("region", "us-east-1"),
},
ncfg: &config.Config{
AlertingConfig: config.AlertingConfig{
@@ -341,9 +339,7 @@ func TestBuildNotifierConfig(t *testing.T) {
},
},
GlobalConfig: config.GlobalConfig{
ExternalLabels: []labels.Label{
{Name: "region", Value: "us-east-1"},
},
ExternalLabels: labels.FromStrings("region", "us-east-1"),
},
},
},
@@ -361,9 +357,7 @@ func TestBuildNotifierConfig(t *testing.T) {
},
},
},
ExternalLabels: []labels.Label{
{Name: "region", Value: "us-east-1"},
},
ExternalLabels: labels.FromStrings("region", "us-east-1"),
},
ncfg: &config.Config{
AlertingConfig: config.AlertingConfig{
@@ -391,9 +385,7 @@ func TestBuildNotifierConfig(t *testing.T) {
},
},
GlobalConfig: config.GlobalConfig{
ExternalLabels: []labels.Label{
{Name: "region", Value: "us-east-1"},
},
ExternalLabels: labels.FromStrings("region", "us-east-1"),
},
},
},

View File

@@ -275,7 +275,7 @@ func TestNotifierSendsUserIDHeader(t *testing.T) {
time.Sleep(10 * time.Millisecond)
}
n.Send(&notifier.Alert{
Labels: labels.Labels{labels.Label{Name: "alertname", Value: "testalert"}},
Labels: labels.FromStrings("alertname", "testalert"),
})
wg.Wait()
@@ -361,11 +361,11 @@ func TestMultiTenantsNotifierSendsUserIDHeader(t *testing.T) {
}
n1.Send(&notifier.Alert{
Labels: labels.Labels{labels.Label{Name: "alertname1", Value: "testalert1"}},
Labels: labels.FromStrings("alertname1", "testalert1"),
})
n2.Send(&notifier.Alert{
Labels: labels.Labels{labels.Label{Name: "alertname2", Value: "testalert2"}},
Labels: labels.FromStrings("alertname2", "testalert2"),
})
// Wait for notifications with a timeout
@@ -1748,8 +1748,8 @@ func TestSendAlerts(t *testing.T) {
{
in: []*promRules.Alert{
{
Labels: []labels.Label{{Name: "l1", Value: "v1"}},
Annotations: []labels.Label{{Name: "a2", Value: "v2"}},
Labels: labels.FromStrings("l1", "v1"),
Annotations: labels.FromStrings("a2", "v2"),
ActiveAt: time.Unix(1, 0),
FiredAt: time.Unix(2, 0),
ValidUntil: time.Unix(3, 0),
@@ -1757,8 +1757,8 @@ func TestSendAlerts(t *testing.T) {
},
exp: []*notifier.Alert{
{
Labels: []labels.Label{{Name: "l1", Value: "v1"}},
Annotations: []labels.Label{{Name: "a2", Value: "v2"}},
Labels: labels.FromStrings("l1", "v1"),
Annotations: labels.FromStrings("a2", "v2"),
StartsAt: time.Unix(2, 0),
EndsAt: time.Unix(3, 0),
GeneratorURL: fmt.Sprintf("http://localhost:8080/explore?left=%%7B%%22queries%%22%%3A%%5B%s%%5D%%7D", escapedExpression),
@@ -1768,8 +1768,8 @@ func TestSendAlerts(t *testing.T) {
{
in: []*promRules.Alert{
{
Labels: []labels.Label{{Name: "l1", Value: "v1"}},
Annotations: []labels.Label{{Name: "a2", Value: "v2"}},
Labels: labels.FromStrings("l1", "v1"),
Annotations: labels.FromStrings("a2", "v2"),
ActiveAt: time.Unix(1, 0),
FiredAt: time.Unix(2, 0),
ResolvedAt: time.Unix(4, 0),
@@ -1777,8 +1777,8 @@ func TestSendAlerts(t *testing.T) {
},
exp: []*notifier.Alert{
{
Labels: []labels.Label{{Name: "l1", Value: "v1"}},
Annotations: []labels.Label{{Name: "a2", Value: "v2"}},
Labels: labels.FromStrings("l1", "v1"),
Annotations: labels.FromStrings("a2", "v2"),
StartsAt: time.Unix(2, 0),
EndsAt: time.Unix(4, 0),
GeneratorURL: fmt.Sprintf("http://localhost:8080/explore?left=%%7B%%22queries%%22%%3A%%5B%s%%5D%%7D", escapedExpression),
@@ -1866,10 +1866,7 @@ func TestRecoverAlertsPostOutage(t *testing.T) {
fn: func(_ bool, _ *storage.SelectHints, _ ...*labels.Matcher) storage.SeriesSet {
return series.NewConcreteSeriesSet([]storage.Series{
series.NewConcreteSeries(
labels.Labels{
{Name: labels.MetricName, Value: "ALERTS_FOR_STATE"},
{Name: labels.AlertName, Value: mockRules["user1"][0].GetRules()[0].Alert},
},
labels.FromStrings(labels.MetricName, "ALERTS_FOR_STATE", labels.AlertName, mockRules["user1"][0].GetRules()[0].Alert),
[]model.SamplePair{{Timestamp: model.Time(downAtTimeMs), Value: model.SampleValue(downAtActiveSec)}},
),
})
@@ -2008,20 +2005,14 @@ func TestRuleGroupAlertsAndSeriesLimit(t *testing.T) {
fn: func(_ bool, _ *storage.SelectHints, _ ...*labels.Matcher) storage.SeriesSet {
return series.NewConcreteSeriesSet([]storage.Series{
series.NewConcreteSeries(
labels.Labels{
{Name: labels.MetricName, Value: "http_requests"},
{Name: labels.InstanceName, Value: "server1"},
},
labels.FromStrings(labels.MetricName, "http_requests", labels.InstanceName, "server1"),
[]model.SamplePair{
{Timestamp: model.Time(seriesStartTime.Add(sampleTimeDiff).UnixMilli()), Value: 100},
{Timestamp: model.Time(currentTime.UnixMilli()), Value: 100},
},
),
series.NewConcreteSeries(
labels.Labels{
{Name: labels.MetricName, Value: "http_requests"},
{Name: labels.InstanceName, Value: "server2"},
},
labels.FromStrings(labels.MetricName, "http_requests", labels.InstanceName, "server2"),
[]model.SamplePair{
{Timestamp: model.Time(seriesStartTime.Add(sampleTimeDiff).UnixMilli()), Value: 100},
{Timestamp: model.Time(currentTime.UnixMilli()), Value: 100},

View File

@@ -21,9 +21,9 @@ import (
const ruleName = "testrule"
func labelsToMatchers(ls labels.Labels) (res []*labels.Matcher) {
for _, l := range ls {
ls.Range(func(l labels.Label) {
res = append(res, labels.MustNewMatcher(labels.MatchEqual, l.Name, l.Value))
}
})
return res
}

View File

@@ -198,10 +198,7 @@ func TestInstance(t *testing.T) {
count := 3
for i := 0; i < count; i++ {
_, err := app.Append(0, labels.Labels{
labels.Label{Name: "__name__", Value: "test"},
labels.Label{Name: "iter", Value: fmt.Sprintf("%v", i)},
}, refTime-int64(i), float64(i))
_, err := app.Append(0, labels.FromStrings("__name__", "test", "iter", fmt.Sprintf("%v", i)), refTime-int64(i), float64(i))
require.NoError(t, err)
}

View File

@@ -42,28 +42,28 @@ func TestStorage_InvalidSeries(t *testing.T) {
_, err = app.Append(0, labels.Labels{}, 0, 0)
require.Error(t, err, "should reject empty labels")
_, err = app.Append(0, labels.Labels{{Name: "a", Value: "1"}, {Name: "a", Value: "2"}}, 0, 0)
_, err = app.Append(0, labels.FromStrings("a", "1", "a", "2"), 0, 0)
require.Error(t, err, "should reject duplicate labels")
// Sanity check: valid series
sRef, err := app.Append(0, labels.Labels{{Name: "a", Value: "1"}}, 0, 0)
sRef, err := app.Append(0, labels.FromStrings("a", "1"), 0, 0)
require.NoError(t, err, "should not reject valid series")
// Exemplars
_, err = app.AppendExemplar(0, nil, exemplar.Exemplar{})
_, err = app.AppendExemplar(0, labels.EmptyLabels(), exemplar.Exemplar{})
require.Error(t, err, "should reject unknown series ref")
e := exemplar.Exemplar{Labels: labels.Labels{{Name: "a", Value: "1"}, {Name: "a", Value: "2"}}}
_, err = app.AppendExemplar(sRef, nil, e)
e := exemplar.Exemplar{Labels: labels.FromStrings("a", "1", "a", "2")}
_, err = app.AppendExemplar(sRef, labels.EmptyLabels(), e)
require.ErrorIs(t, err, tsdb.ErrInvalidExemplar, "should reject duplicate labels")
e = exemplar.Exemplar{Labels: labels.Labels{{Name: "a_somewhat_long_trace_id", Value: "nYJSNtFrFTY37VR7mHzEE/LIDt7cdAQcuOzFajgmLDAdBSRHYPDzrxhMA4zz7el8naI/AoXFv9/e/G0vcETcIoNUi3OieeLfaIRQci2oa"}}}
_, err = app.AppendExemplar(sRef, nil, e)
e = exemplar.Exemplar{Labels: labels.FromStrings("a_somewhat_long_trace_id", "nYJSNtFrFTY37VR7mHzEE/LIDt7cdAQcuOzFajgmLDAdBSRHYPDzrxhMA4zz7el8naI/AoXFv9/e/G0vcETcIoNUi3OieeLfaIRQci2oa")}
_, err = app.AppendExemplar(sRef, labels.EmptyLabels(), e)
require.ErrorIs(t, err, storage.ErrExemplarLabelLength, "should reject too long label length")
// Sanity check: valid exemplars
e = exemplar.Exemplar{Labels: labels.Labels{{Name: "a", Value: "1"}}, Value: 20, Ts: 10, HasTs: true}
_, err = app.AppendExemplar(sRef, nil, e)
e = exemplar.Exemplar{Labels: labels.FromStrings("a", "1"), Value: 20, Ts: 10, HasTs: true}
_, err = app.AppendExemplar(sRef, labels.EmptyLabels(), e)
require.NoError(t, err, "should not reject valid exemplars")
}
@@ -386,7 +386,7 @@ type series struct {
func (s *series) Write(t *testing.T, app storage.Appender) {
t.Helper()
lbls := labels.FromMap(map[string]string{"__name__": s.name})
lbls := labels.FromStrings("__name__", s.name)
offset := 0
if s.ref == nil {
@@ -407,7 +407,7 @@ func (s *series) Write(t *testing.T, app storage.Appender) {
sRef := *s.ref
for _, exemplar := range s.exemplars {
var err error
sRef, err = app.AppendExemplar(sRef, nil, exemplar)
sRef, err = app.AppendExemplar(sRef, labels.EmptyLabels(), exemplar)
require.NoError(t, err)
}
}
@@ -499,8 +499,8 @@ func buildSeries(nameSlice []string) seriesList {
name: n,
samples: []sample{{int64(i), float64(i * 10.0)}, {int64(i * 10), float64(i * 100.0)}},
exemplars: []exemplar.Exemplar{
{Labels: labels.Labels{{Name: "foobar", Value: "barfoo"}}, Value: float64(i * 10.0), Ts: int64(i), HasTs: true},
{Labels: labels.Labels{{Name: "lorem", Value: "ipsum"}}, Value: float64(i * 100.0), Ts: int64(i * 10), HasTs: true},
{Labels: labels.FromStrings("foobar", "barfoo"), Value: float64(i * 10.0), Ts: int64(i), HasTs: true},
{Labels: labels.FromStrings("lorem", "ipsum"), Value: float64(i * 100.0), Ts: int64(i * 10), HasTs: true},
},
})
}

View File

@@ -41,9 +41,9 @@ func RoundToMilliseconds(from, through time.Time) (model.Time, model.Time) {
// LabelsToMetric converts a Labels to Metric
// Don't do this on any performance sensitive paths.
func LabelsToMetric(ls labels.Labels) model.Metric {
m := make(model.Metric, len(ls))
for _, l := range ls {
m := make(model.Metric, ls.Len())
ls.Range(func(l labels.Label) {
m[model.LabelName(l.Name)] = model.LabelValue(l.Value)
}
})
return m
}

View File

@@ -727,7 +727,7 @@ func (r *UserRegistries) BuildMetricFamiliesPerUser(labelTransformFn MetricLabel
// FromLabelPairsToLabels converts dto.LabelPair into labels.Labels.
func FromLabelPairsToLabels(pairs []*dto.LabelPair) labels.Labels {
builder := labels.NewBuilder(nil)
builder := labels.NewBuilder(labels.EmptyLabels())
for _, pair := range pairs {
builder.Set(pair.GetName(), pair.GetValue())
}
@@ -774,7 +774,7 @@ func GetLabels(c prometheus.Collector, filter map[string]string) ([]labels.Label
errs := tsdb_errors.NewMulti()
var result []labels.Labels
dtoMetric := &dto.Metric{}
lbls := labels.NewBuilder(nil)
lbls := labels.NewBuilder(labels.EmptyLabels())
nextMetric:
for m := range ch {
@@ -785,7 +785,7 @@ nextMetric:
continue
}
lbls.Reset(nil)
lbls.Reset(labels.EmptyLabels())
for _, lp := range dtoMetric.Label {
n := lp.GetName()
v := lp.GetValue()