Prometheus: Small improvements to the custom client (#51709)

* Add suggestions

* Fix tests

pkg/tsdb/prometheus/client/client.go

@@ -138,10 +138,13 @@ func createRequest(ctx context.Context, method string, u *url.URL, body []byte,
 	if header != nil {
 		request.Header = header
 	}
-	// This may not be true but right now we don't have more information here and seems like we send just this type
-	// of encoding right now if it is a POST
 	if strings.ToUpper(method) == http.MethodPost {
+		// This may not be true but right now we don't have more information here and seems like we send just this type
+		// of encoding right now if it is a POST
 		request.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+		// This allows transport to retry request. See https://github.com/prometheus/client_golang/pull/1022
+		// It's set to nil so it is not actually sent over the wire, just used in Go http lib to retry requests.
+		request.Header["Idempotency-Key"] = nil
 	}
 	return request, nil
 }

pkg/tsdb/prometheus/prometheus_test.go

@@ -95,7 +95,14 @@ func TestService(t *testing.T) {
 				sender := &fakeSender{}
 				err := service.CallResource(context.Background(), req, sender)
 				require.NoError(t, err)
-				require.Equal(t, http.Header{"Content-Type": {"application/x-www-form-urlencoded"}, "foo": {"bar"}}, httpProvider.Roundtripper.Req.Header)
+				require.Equal(
+					t,
+					http.Header{
+						"Content-Type":    {"application/x-www-form-urlencoded"},
+						"Idempotency-Key": []string(nil),
+						"foo":             {"bar"},
+					},
+					httpProvider.Roundtripper.Req.Header)
 				require.Equal(t, http.MethodPost, httpProvider.Roundtripper.Req.Method)
 				body, err := io.ReadAll(httpProvider.Roundtripper.Req.Body)
 				require.NoError(t, err)

pkg/tsdb/prometheus/resource/resource.go

@@ -1,9 +1,9 @@
 package resource
 
 import (
+	"bytes"
 	"context"
 	"fmt"
-	"io/ioutil"
 	"net/http"
 
 	"github.com/grafana/grafana-plugin-sdk-go/backend"
@@ -83,14 +83,17 @@ func (r *Resource) Execute(ctx context.Context, req *backend.CallResourceRequest
 		}
 	}()
 
-	data, err := ioutil.ReadAll(resp.Body)
+	var buf bytes.Buffer
+	// Should be more efficient than ReadAll. See https://github.com/prometheus/client_golang/pull/976
+	_, err = buf.ReadFrom(resp.Body)
+	body := buf.Bytes()
 	if err != nil {
 		return nil, err
 	}
 	callResponse := &backend.CallResourceResponse{
 		Status:  resp.StatusCode,
 		Headers: resp.Header,
-		Body:    data,
+		Body:    body,
 	}
 
 	return callResponse, err