diff --git a/pkg/tsdb/elasticsearch/client/client.go b/pkg/tsdb/elasticsearch/client/client.go index 746e670dc78..b9ddc051469 100644 --- a/pkg/tsdb/elasticsearch/client/client.go +++ b/pkg/tsdb/elasticsearch/client/client.go @@ -344,11 +344,19 @@ func processHits(dec *json.Decoder, sr *SearchResponse) error { return err } - if tok == "hits" { + switch tok { + case "hits": if err := streamHitsArray(dec, sr); err != nil { return err } - } else { + case "total": + var total *SearchResponseHitsTotal + err := dec.Decode(&total) + if err != nil { + return err + } + sr.Hits.Total = total + default: // ignore these fields as they are not used in the current implementation err := skipUnknownField(dec) if err != nil { diff --git a/pkg/tsdb/elasticsearch/client/models.go b/pkg/tsdb/elasticsearch/client/models.go index e18fad67f35..c8648f0bdeb 100644 --- a/pkg/tsdb/elasticsearch/client/models.go +++ b/pkg/tsdb/elasticsearch/client/models.go @@ -44,9 +44,15 @@ func (r *SearchRequest) MarshalJSON() ([]byte, error) { return json.Marshal(root) } +type SearchResponseHitsTotal struct { + Value int `json:"value"` + Relation string `json:"relation"` +} + // SearchResponseHits represents search response hits type SearchResponseHits struct { - Hits []map[string]interface{} + Hits []map[string]interface{} + Total *SearchResponseHitsTotal `json:"total"` } // SearchResponse represents a search response diff --git a/pkg/tsdb/elasticsearch/response_parser.go b/pkg/tsdb/elasticsearch/response_parser.go index 541006fedc2..b38b2c1cc07 100644 --- a/pkg/tsdb/elasticsearch/response_parser.go +++ b/pkg/tsdb/elasticsearch/response_parser.go @@ -208,7 +208,12 @@ func processLogsResponse(res *es.SearchResponse, target *Query, configuredFields frames := data.Frames{} frame := data.NewFrame("", fields...) setPreferredVisType(frame, data.VisTypeLogs) - setLogsCustomMeta(frame, searchWords, stringToIntWithDefaultValue(target.Metrics[0].Settings.Get("limit").MustString(), defaultSize)) + + var total int + if res.Hits.Total != nil { + total = res.Hits.Total.Value + } + setLogsCustomMeta(frame, searchWords, stringToIntWithDefaultValue(target.Metrics[0].Settings.Get("limit").MustString(), defaultSize), total) frames = append(frames, frame) queryRes.Frames = frames @@ -1192,7 +1197,7 @@ func setPreferredVisType(frame *data.Frame, visType data.VisType) { frame.Meta.PreferredVisualization = visType } -func setLogsCustomMeta(frame *data.Frame, searchWords map[string]bool, limit int) { +func setLogsCustomMeta(frame *data.Frame, searchWords map[string]bool, limit int, total int) { i := 0 searchWordsList := make([]string, len(searchWords)) for searchWord := range searchWords { @@ -1212,6 +1217,7 @@ func setLogsCustomMeta(frame *data.Frame, searchWords map[string]bool, limit int frame.Meta.Custom = map[string]interface{}{ "searchWords": searchWordsList, "limit": limit, + "total": total, } } diff --git a/pkg/tsdb/elasticsearch/response_parser_test.go b/pkg/tsdb/elasticsearch/response_parser_test.go index a03c5068ed8..85239537bad 100644 --- a/pkg/tsdb/elasticsearch/response_parser_test.go +++ b/pkg/tsdb/elasticsearch/response_parser_test.go @@ -46,6 +46,7 @@ func TestProcessLogsResponse(t *testing.T) { { "aggregations": {}, "hits": { + "total": { "value": 2 }, "hits": [ { "_id": "fdsfs", @@ -107,7 +108,7 @@ func TestProcessLogsResponse(t *testing.T) { logsFrame := frames[0] meta := logsFrame.Meta - require.Equal(t, map[string]any{"searchWords": []string{"hello", "message"}, "limit": 500}, meta.Custom) + require.Equal(t, map[string]any{"searchWords": []string{"hello", "message"}, "limit": 500, "total": 2}, meta.Custom) require.Equal(t, data.VisTypeLogs, string(meta.PreferredVisualization)) logsFieldMap := make(map[string]*data.Field) @@ -431,6 +432,7 @@ func TestProcessLogsResponse(t *testing.T) { require.Equal(t, map[string]any{ "searchWords": []string{"hello", "message"}, "limit": 500, + "total": 109, }, customMeta) }) } @@ -703,7 +705,7 @@ func TestProcessRawDocumentResponse(t *testing.T) { "responses": [ { "hits": { - "total": 100, + "total": { "value": 100 }, "hits": [ { "_id": "1", @@ -3239,7 +3241,7 @@ func TestParseResponse(t *testing.T) { }, { "hits": { - "total": 2, + "total": { "value": 2 }, "hits": [ { "_id": "5", diff --git a/pkg/tsdb/elasticsearch/testdata_response/logs.a.golden.jsonc b/pkg/tsdb/elasticsearch/testdata_response/logs.a.golden.jsonc index 105279a888e..7d5a673da01 100644 --- a/pkg/tsdb/elasticsearch/testdata_response/logs.a.golden.jsonc +++ b/pkg/tsdb/elasticsearch/testdata_response/logs.a.golden.jsonc @@ -10,7 +10,8 @@ // "searchWords": [ // "hello", // "message" -// ] +// ], +// "total": 81 // }, // "preferredVisualisationType": "logs" // } @@ -45,7 +46,8 @@ "searchWords": [ "hello", "message" - ] + ], + "total": 81 }, "preferredVisualisationType": "logs" }, diff --git a/public/app/features/logs/logsModel.test.ts b/public/app/features/logs/logsModel.test.ts index 3c7b599c7f6..62f692e4846 100644 --- a/public/app/features/logs/logsModel.test.ts +++ b/public/app/features/logs/logsModel.test.ts @@ -34,6 +34,7 @@ import { filterLogLevels, getSeriesProperties, LIMIT_LABEL, + TOTAL_LABEL, logRowToSingleRowDataFrame, logSeriesToLogsModel, queryLogsSample, @@ -492,6 +493,52 @@ describe('dataFrameToLogsModel', () => { }); }); + it('given one series with total as custom meta property should return correct total', () => { + const series: DataFrame[] = [ + createDataFrame({ + fields: [], + meta: { + custom: { + total: 9999, + }, + }, + }), + ]; + const logsModel = dataFrameToLogsModel(series, 1); + expect(logsModel.meta![0]).toMatchObject({ + label: TOTAL_LABEL, + value: 9999, + kind: LogsMetaKind.Number, + }); + }); + + it('given multiple series with total as custom meta property should return correct total', () => { + const series: DataFrame[] = [ + createDataFrame({ + fields: [], + meta: { + custom: { + total: 4, + }, + }, + }), + createDataFrame({ + fields: [], + meta: { + custom: { + total: 5, + }, + }, + }), + ]; + const logsModel = dataFrameToLogsModel(series, 1); + expect(logsModel.meta![0]).toMatchObject({ + label: TOTAL_LABEL, + value: 9, + kind: LogsMetaKind.Number, + }); + }); + it('should return the expected meta when the line limit is reached', () => { const series: DataFrame[] = getTestDataFrame(); series[0].meta = { diff --git a/public/app/features/logs/logsModel.ts b/public/app/features/logs/logsModel.ts index 7e75684d423..9c1c266cea6 100644 --- a/public/app/features/logs/logsModel.ts +++ b/public/app/features/logs/logsModel.ts @@ -51,6 +51,7 @@ import { createLogRowsMap, getLogLevel, getLogLevelFromKey, sortInAscendingOrder export const LIMIT_LABEL = 'Line limit'; export const COMMON_LABELS = 'Common labels'; +export const TOTAL_LABEL = 'Total lines'; export const LogLevelColor = { [LogLevel.critical]: colors[7], @@ -492,6 +493,15 @@ export function logSeriesToLogsModel( }); } + const totalValue = logSeries.reduce((acc, series) => (acc += series.meta?.custom?.total), 0); + if (totalValue > 0) { + meta.push({ + label: TOTAL_LABEL, + value: totalValue, + kind: LogsMetaKind.Number, + }); + } + let totalBytes = 0; const queriesVisited: { [refId: string]: boolean } = {}; // To add just 1 error message