Logs panel: Add meta field to show total hits; add total hits to ElasticSearch plugin response (#104117)

* feat: Show total amount of hits in Elastic Search query

* Add test with multiple series.

pkg/tsdb/elasticsearch/client/client.go

@@ -344,11 +344,19 @@ func processHits(dec *json.Decoder, sr *SearchResponse) error {
 			return err
 		}
 
-		if tok == "hits" {
+		switch tok {
+		case "hits":
 			if err := streamHitsArray(dec, sr); err != nil {
 				return err
 			}
-		} else {
+		case "total":
+			var total *SearchResponseHitsTotal
+			err := dec.Decode(&total)
+			if err != nil {
+				return err
+			}
+			sr.Hits.Total = total
+		default:
 			// ignore these fields as they are not used in the current implementation
 			err := skipUnknownField(dec)
 			if err != nil {

pkg/tsdb/elasticsearch/client/models.go

@@ -44,9 +44,15 @@ func (r *SearchRequest) MarshalJSON() ([]byte, error) {
 	return json.Marshal(root)
 }
 
+type SearchResponseHitsTotal struct {
+	Value    int    `json:"value"`
+	Relation string `json:"relation"`
+}
+
 // SearchResponseHits represents search response hits
 type SearchResponseHits struct {
 	Hits  []map[string]interface{}
+	Total *SearchResponseHitsTotal `json:"total"`
 }
 
 // SearchResponse represents a search response

pkg/tsdb/elasticsearch/response_parser.go

@@ -208,7 +208,12 @@ func processLogsResponse(res *es.SearchResponse, target *Query, configuredFields
 	frames := data.Frames{}
 	frame := data.NewFrame("", fields...)
 	setPreferredVisType(frame, data.VisTypeLogs)
-	setLogsCustomMeta(frame, searchWords, stringToIntWithDefaultValue(target.Metrics[0].Settings.Get("limit").MustString(), defaultSize))
+
+	var total int
+	if res.Hits.Total != nil {
+		total = res.Hits.Total.Value
+	}
+	setLogsCustomMeta(frame, searchWords, stringToIntWithDefaultValue(target.Metrics[0].Settings.Get("limit").MustString(), defaultSize), total)
 	frames = append(frames, frame)
 	queryRes.Frames = frames
 
@@ -1192,7 +1197,7 @@ func setPreferredVisType(frame *data.Frame, visType data.VisType) {
 	frame.Meta.PreferredVisualization = visType
 }
 
-func setLogsCustomMeta(frame *data.Frame, searchWords map[string]bool, limit int) {
+func setLogsCustomMeta(frame *data.Frame, searchWords map[string]bool, limit int, total int) {
 	i := 0
 	searchWordsList := make([]string, len(searchWords))
 	for searchWord := range searchWords {
@@ -1212,6 +1217,7 @@ func setLogsCustomMeta(frame *data.Frame, searchWords map[string]bool, limit int
 	frame.Meta.Custom = map[string]interface{}{
 		"searchWords": searchWordsList,
 		"limit":       limit,
+		"total":       total,
 	}
 }
 

pkg/tsdb/elasticsearch/response_parser_test.go

@@ -46,6 +46,7 @@ func TestProcessLogsResponse(t *testing.T) {
 					  {
 						"aggregations": {},
 						"hits": {
+						  "total": { "value": 2 },
 						  "hits": [
 							{
 							  "_id": "fdsfs",
@@ -107,7 +108,7 @@ func TestProcessLogsResponse(t *testing.T) {
 			logsFrame := frames[0]
 
 			meta := logsFrame.Meta
-			require.Equal(t, map[string]any{"searchWords": []string{"hello", "message"}, "limit": 500}, meta.Custom)
+			require.Equal(t, map[string]any{"searchWords": []string{"hello", "message"}, "limit": 500, "total": 2}, meta.Custom)
 			require.Equal(t, data.VisTypeLogs, string(meta.PreferredVisualization))
 
 			logsFieldMap := make(map[string]*data.Field)
@@ -431,6 +432,7 @@ func TestProcessLogsResponse(t *testing.T) {
 		require.Equal(t, map[string]any{
 			"searchWords": []string{"hello", "message"},
 			"limit":       500,
+			"total":       109,
 		}, customMeta)
 	})
 }
@@ -703,7 +705,7 @@ func TestProcessRawDocumentResponse(t *testing.T) {
 		"responses": [
 			{
 			"hits": {
-				"total": 100,
+				"total": { "value": 100 },
 				"hits": [
 				{
 					"_id": "1",
@@ -3239,7 +3241,7 @@ func TestParseResponse(t *testing.T) {
 				  },
 				  {
 					"hits": {
-					  "total": 2,
+					  "total": { "value": 2 },
 					  "hits": [
 						{
 						  "_id": "5",

pkg/tsdb/elasticsearch/testdata_response/logs.a.golden.jsonc

@@ -10,7 +10,8 @@
 //          "searchWords": [
 //              "hello",
 //              "message"
-//          ]
+//          ],
+//          "total": 81
 //      },
 //      "preferredVisualisationType": "logs"
 //  }
@@ -45,7 +46,8 @@
             "searchWords": [
               "hello",
               "message"
-            ]
+            ],
+            "total": 81
           },
           "preferredVisualisationType": "logs"
         },

public/app/features/logs/logsModel.test.ts

@@ -34,6 +34,7 @@ import {
   filterLogLevels,
   getSeriesProperties,
   LIMIT_LABEL,
+  TOTAL_LABEL,
   logRowToSingleRowDataFrame,
   logSeriesToLogsModel,
   queryLogsSample,
@@ -492,6 +493,52 @@ describe('dataFrameToLogsModel', () => {
     });
   });
 
+  it('given one series with total as custom meta property should return correct total', () => {
+    const series: DataFrame[] = [
+      createDataFrame({
+        fields: [],
+        meta: {
+          custom: {
+            total: 9999,
+          },
+        },
+      }),
+    ];
+    const logsModel = dataFrameToLogsModel(series, 1);
+    expect(logsModel.meta![0]).toMatchObject({
+      label: TOTAL_LABEL,
+      value: 9999,
+      kind: LogsMetaKind.Number,
+    });
+  });
+
+  it('given multiple series with total as custom meta property should return correct total', () => {
+    const series: DataFrame[] = [
+      createDataFrame({
+        fields: [],
+        meta: {
+          custom: {
+            total: 4,
+          },
+        },
+      }),
+      createDataFrame({
+        fields: [],
+        meta: {
+          custom: {
+            total: 5,
+          },
+        },
+      }),
+    ];
+    const logsModel = dataFrameToLogsModel(series, 1);
+    expect(logsModel.meta![0]).toMatchObject({
+      label: TOTAL_LABEL,
+      value: 9,
+      kind: LogsMetaKind.Number,
+    });
+  });
+
   it('should return the expected meta when the line limit is reached', () => {
     const series: DataFrame[] = getTestDataFrame();
     series[0].meta = {

public/app/features/logs/logsModel.ts

@@ -51,6 +51,7 @@ import { createLogRowsMap, getLogLevel, getLogLevelFromKey, sortInAscendingOrder
 
 export const LIMIT_LABEL = 'Line limit';
 export const COMMON_LABELS = 'Common labels';
+export const TOTAL_LABEL = 'Total lines';
 
 export const LogLevelColor = {
   [LogLevel.critical]: colors[7],
@@ -492,6 +493,15 @@ export function logSeriesToLogsModel(
     });
   }
 
+  const totalValue = logSeries.reduce((acc, series) => (acc += series.meta?.custom?.total), 0);
+  if (totalValue > 0) {
+    meta.push({
+      label: TOTAL_LABEL,
+      value: totalValue,
+      kind: LogsMetaKind.Number,
+    });
+  }
+
   let totalBytes = 0;
   const queriesVisited: { [refId: string]: boolean } = {};
   // To add just 1 error message