diff --git a/conf/defaults.ini b/conf/defaults.ini
index 499c44451c6..a1d44ee4aa6 100644
--- a/conf/defaults.ini
+++ b/conf/defaults.ini
@@ -172,6 +172,9 @@ idle_conn_timeout_seconds = 90
# If enabled and user is not anonymous, data proxy will add X-Grafana-User header with username into the request.
send_user_header = false
+# Limit the amount of bytes that will be read/accepted from responses of outgoing HTTP requests.
+response_limit = 0
+
#################################### Analytics ###########################
[analytics]
# Server reporting, sends usage counters to stats.grafana.org every 24 hours.
diff --git a/conf/sample.ini b/conf/sample.ini
index 3b3c3c16df5..7d27f359ffb 100644
--- a/conf/sample.ini
+++ b/conf/sample.ini
@@ -178,6 +178,9 @@
# If enabled and user is not anonymous, data proxy will add X-Grafana-User header with username into the request, default is false.
;send_user_header = false
+# Limit the amount of bytes that will be read/accepted from responses of outgoing HTTP requests.
+;response_limit = 0
+
#################################### Analytics ####################################
[analytics]
# Server reporting, sends usage counters to stats.grafana.org every 24 hours.
@@ -483,14 +486,14 @@
;auth_url = https://foo.bar/login/oauth/authorize
;token_url = https://foo.bar/login/oauth/access_token
;api_url = https://foo.bar/user
-;teams_url =
+;teams_url =
;allowed_domains =
;team_ids =
;allowed_organizations =
;role_attribute_path =
;role_attribute_strict = false
;groups_attribute_path =
-;team_ids_attribute_path =
+;team_ids_attribute_path =
;tls_skip_verify_insecure = false
;tls_client_cert =
;tls_client_key =
diff --git a/docs/sources/administration/configuration.md b/docs/sources/administration/configuration.md
index bd7201c28ee..902f704ebb5 100644
--- a/docs/sources/administration/configuration.md
+++ b/docs/sources/administration/configuration.md
@@ -435,6 +435,10 @@ The length of time that Grafana maintains idle connections before closing them.
If enabled and user is not anonymous, data proxy will add X-Grafana-User header with username into the request. Default is `false`.
+### response_limit
+
+Limits the amount of bytes that will be read/accepted from responses of outgoing HTTP requests. Default is `0` which means disabled.
+
## [analytics]
diff --git a/pkg/infra/httpclient/httpclientprovider/http_client_provider.go b/pkg/infra/httpclient/httpclientprovider/http_client_provider.go
index 1fe5179cf0a..f1b4ec367aa 100644
--- a/pkg/infra/httpclient/httpclientprovider/http_client_provider.go
+++ b/pkg/infra/httpclient/httpclientprovider/http_client_provider.go
@@ -25,6 +25,7 @@ func New(cfg *setting.Cfg) *sdkhttpclient.Provider {
SetUserAgentMiddleware(userAgent),
sdkhttpclient.BasicAuthenticationMiddleware(),
sdkhttpclient.CustomHeadersMiddleware(),
+ ResponseLimitMiddleware(cfg.ResponseLimit),
}
if cfg.SigV4AuthEnabled {
diff --git a/pkg/infra/httpclient/httpclientprovider/http_client_provider_test.go b/pkg/infra/httpclient/httpclientprovider/http_client_provider_test.go
index 6b6c310368b..c9a3bdd4d76 100644
--- a/pkg/infra/httpclient/httpclientprovider/http_client_provider_test.go
+++ b/pkg/infra/httpclient/httpclientprovider/http_client_provider_test.go
@@ -22,12 +22,13 @@ func TestHTTPClientProvider(t *testing.T) {
_ = New(&setting.Cfg{SigV4AuthEnabled: false})
require.Len(t, providerOpts, 1)
o := providerOpts[0]
- require.Len(t, o.Middlewares, 5)
+ require.Len(t, o.Middlewares, 6)
require.Equal(t, TracingMiddlewareName, o.Middlewares[0].(sdkhttpclient.MiddlewareName).MiddlewareName())
require.Equal(t, DataSourceMetricsMiddlewareName, o.Middlewares[1].(sdkhttpclient.MiddlewareName).MiddlewareName())
require.Equal(t, SetUserAgentMiddlewareName, o.Middlewares[2].(sdkhttpclient.MiddlewareName).MiddlewareName())
require.Equal(t, sdkhttpclient.BasicAuthenticationMiddlewareName, o.Middlewares[3].(sdkhttpclient.MiddlewareName).MiddlewareName())
require.Equal(t, sdkhttpclient.CustomHeadersMiddlewareName, o.Middlewares[4].(sdkhttpclient.MiddlewareName).MiddlewareName())
+ require.Equal(t, ResponseLimitMiddlewareName, o.Middlewares[5].(sdkhttpclient.MiddlewareName).MiddlewareName())
})
t.Run("When creating new provider and SigV4 is enabled should apply expected middleware", func(t *testing.T) {
@@ -43,12 +44,13 @@ func TestHTTPClientProvider(t *testing.T) {
_ = New(&setting.Cfg{SigV4AuthEnabled: true})
require.Len(t, providerOpts, 1)
o := providerOpts[0]
- require.Len(t, o.Middlewares, 6)
+ require.Len(t, o.Middlewares, 7)
require.Equal(t, TracingMiddlewareName, o.Middlewares[0].(sdkhttpclient.MiddlewareName).MiddlewareName())
require.Equal(t, DataSourceMetricsMiddlewareName, o.Middlewares[1].(sdkhttpclient.MiddlewareName).MiddlewareName())
require.Equal(t, SetUserAgentMiddlewareName, o.Middlewares[2].(sdkhttpclient.MiddlewareName).MiddlewareName())
require.Equal(t, sdkhttpclient.BasicAuthenticationMiddlewareName, o.Middlewares[3].(sdkhttpclient.MiddlewareName).MiddlewareName())
require.Equal(t, sdkhttpclient.CustomHeadersMiddlewareName, o.Middlewares[4].(sdkhttpclient.MiddlewareName).MiddlewareName())
- require.Equal(t, SigV4MiddlewareName, o.Middlewares[5].(sdkhttpclient.MiddlewareName).MiddlewareName())
+ require.Equal(t, ResponseLimitMiddlewareName, o.Middlewares[5].(sdkhttpclient.MiddlewareName).MiddlewareName())
+ require.Equal(t, SigV4MiddlewareName, o.Middlewares[6].(sdkhttpclient.MiddlewareName).MiddlewareName())
})
}
diff --git a/pkg/infra/httpclient/httpclientprovider/response_limit_middleware.go b/pkg/infra/httpclient/httpclientprovider/response_limit_middleware.go
new file mode 100644
index 00000000000..c17c70b3516
--- /dev/null
+++ b/pkg/infra/httpclient/httpclientprovider/response_limit_middleware.go
@@ -0,0 +1,28 @@
+package httpclientprovider
+
+import (
+ "net/http"
+
+ sdkhttpclient "github.com/grafana/grafana-plugin-sdk-go/backend/httpclient"
+ "github.com/grafana/grafana/pkg/infra/httpclient"
+)
+
+// ResponseLimitMiddlewareName is the middleware name used by ResponseLimitMiddleware.
+const ResponseLimitMiddlewareName = "response-limit"
+
+func ResponseLimitMiddleware(limit int64) sdkhttpclient.Middleware {
+ return sdkhttpclient.NamedMiddlewareFunc(ResponseLimitMiddlewareName, func(opts sdkhttpclient.Options, next http.RoundTripper) http.RoundTripper {
+ if limit <= 0 {
+ return next
+ }
+ return sdkhttpclient.RoundTripperFunc(func(req *http.Request) (*http.Response, error) {
+ res, err := next.RoundTrip(req)
+ if err != nil {
+ return nil, err
+ }
+
+ res.Body = httpclient.MaxBytesReader(res.Body, limit)
+ return res, nil
+ })
+ })
+}
diff --git a/pkg/infra/httpclient/httpclientprovider/response_limit_middleware_test.go b/pkg/infra/httpclient/httpclientprovider/response_limit_middleware_test.go
new file mode 100644
index 00000000000..4a4064a8f44
--- /dev/null
+++ b/pkg/infra/httpclient/httpclientprovider/response_limit_middleware_test.go
@@ -0,0 +1,60 @@
+package httpclientprovider
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "io/ioutil"
+ "net/http"
+ "strings"
+ "testing"
+
+ "github.com/grafana/grafana-plugin-sdk-go/backend/httpclient"
+ "github.com/stretchr/testify/require"
+)
+
+func TestResponseLimitMiddleware(t *testing.T) {
+ tcs := []struct {
+ limit int64
+ bodyLength int
+ body string
+ err error
+ }{
+ {limit: 1, bodyLength: 1, body: "d", err: errors.New("error: http: response body too large, response limit is set to: 1")},
+ {limit: 1000000, bodyLength: 5, body: "dummy", err: nil},
+ {limit: 0, bodyLength: 5, body: "dummy", err: nil},
+ }
+ for _, tc := range tcs {
+ t.Run(fmt.Sprintf("Test ResponseLimitMiddleware with limit: %d", tc.limit), func(t *testing.T) {
+ finalRoundTripper := httpclient.RoundTripperFunc(func(req *http.Request) (*http.Response, error) {
+ return &http.Response{StatusCode: http.StatusOK, Request: req, Body: ioutil.NopCloser(strings.NewReader("dummy"))}, nil
+ })
+
+ mw := ResponseLimitMiddleware(tc.limit)
+ rt := mw.CreateMiddleware(httpclient.Options{}, finalRoundTripper)
+ require.NotNil(t, rt)
+ middlewareName, ok := mw.(httpclient.MiddlewareName)
+ require.True(t, ok)
+ require.Equal(t, ResponseLimitMiddlewareName, middlewareName.MiddlewareName())
+
+ ctx := context.Background()
+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://test.com/query", nil)
+ require.NoError(t, err)
+ res, err := rt.RoundTrip(req)
+ require.NoError(t, err)
+ require.NotNil(t, res)
+ require.NotNil(t, res.Body)
+ require.NoError(t, res.Body.Close())
+
+ bodyBytes, err := ioutil.ReadAll(res.Body)
+ if err != nil {
+ require.EqualError(t, tc.err, err.Error())
+ } else {
+ require.NoError(t, tc.err)
+ }
+
+ require.Len(t, bodyBytes, tc.bodyLength)
+ require.Equal(t, string(bodyBytes), tc.body)
+ })
+ }
+}
diff --git a/pkg/infra/httpclient/max_bytes_reader.go b/pkg/infra/httpclient/max_bytes_reader.go
new file mode 100644
index 00000000000..9bbdce1e375
--- /dev/null
+++ b/pkg/infra/httpclient/max_bytes_reader.go
@@ -0,0 +1,66 @@
+package httpclient
+
+import (
+ "errors"
+ "fmt"
+ "io"
+)
+
+// Similar implementation to http/net MaxBytesReader
+// https://pkg.go.dev/net/http#MaxBytesReader
+// What's happening differently here, is that the field that
+// is limited is the response and not the request, thus
+// the error handling/message needed to be accurate.
+
+// ErrResponseBodyTooLarge indicates response body is too large
+var ErrResponseBodyTooLarge = errors.New("http: response body too large")
+
+// MaxBytesReader is similar to io.LimitReader but is intended for
+// limiting the size of incoming request bodies. In contrast to
+// io.LimitReader, MaxBytesReader's result is a ReadCloser, returns a
+// non-EOF error for a Read beyond the limit, and closes the
+// underlying reader when its Close method is called.
+//
+// MaxBytesReader prevents clients from accidentally or maliciously
+// sending a large request and wasting server resources.
+func MaxBytesReader(r io.ReadCloser, n int64) io.ReadCloser {
+ return &maxBytesReader{r: r, n: n}
+}
+
+type maxBytesReader struct {
+ r io.ReadCloser // underlying reader
+ n int64 // max bytes remaining
+ err error // sticky error
+}
+
+func (l *maxBytesReader) Read(p []byte) (n int, err error) {
+ if l.err != nil {
+ return 0, l.err
+ }
+ if len(p) == 0 {
+ return 0, nil
+ }
+ // If they asked for a 32KB byte read but only 5 bytes are
+ // remaining, no need to read 32KB. 6 bytes will answer the
+ // question of the whether we hit the limit or go past it.
+ if int64(len(p)) > l.n+1 {
+ p = p[:l.n+1]
+ }
+ n, err = l.r.Read(p)
+
+ if int64(n) <= l.n {
+ l.n -= int64(n)
+ l.err = err
+ return n, err
+ }
+
+ n = int(l.n)
+ l.n = 0
+
+ l.err = fmt.Errorf("error: %w, response limit is set to: %d", ErrResponseBodyTooLarge, n)
+ return n, l.err
+}
+
+func (l *maxBytesReader) Close() error {
+ return l.r.Close()
+}
diff --git a/pkg/infra/httpclient/max_bytes_reader_test.go b/pkg/infra/httpclient/max_bytes_reader_test.go
new file mode 100644
index 00000000000..13742107550
--- /dev/null
+++ b/pkg/infra/httpclient/max_bytes_reader_test.go
@@ -0,0 +1,40 @@
+package httpclient
+
+import (
+ "errors"
+ "fmt"
+ "io/ioutil"
+ "strings"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+func TestMaxBytesReader(t *testing.T) {
+ tcs := []struct {
+ limit int64
+ bodyLength int
+ body string
+ err error
+ }{
+ {limit: 1, bodyLength: 1, body: "d", err: errors.New("error: http: response body too large, response limit is set to: 1")},
+ {limit: 1000000, bodyLength: 5, body: "dummy", err: nil},
+ {limit: 0, bodyLength: 0, body: "", err: errors.New("error: http: response body too large, response limit is set to: 0")},
+ }
+ for _, tc := range tcs {
+ t.Run(fmt.Sprintf("Test MaxBytesReader with limit: %d", tc.limit), func(t *testing.T) {
+ body := ioutil.NopCloser(strings.NewReader("dummy"))
+ readCloser := MaxBytesReader(body, tc.limit)
+
+ bodyBytes, err := ioutil.ReadAll(readCloser)
+ if err != nil {
+ require.EqualError(t, tc.err, err.Error())
+ } else {
+ require.NoError(t, tc.err)
+ }
+
+ require.Len(t, bodyBytes, tc.bodyLength)
+ require.Equal(t, string(bodyBytes), tc.body)
+ })
+ }
+}
diff --git a/pkg/setting/setting.go b/pkg/setting/setting.go
index b89beaec0e6..c820fafaf84 100644
--- a/pkg/setting/setting.go
+++ b/pkg/setting/setting.go
@@ -326,6 +326,7 @@ type Cfg struct {
DataProxyMaxIdleConns int
DataProxyKeepAlive int
DataProxyIdleConnTimeout int
+ ResponseLimit int64
// DistributedCache
RemoteCacheOptions *RemoteCacheOptions
diff --git a/pkg/setting/setting_data_proxy.go b/pkg/setting/setting_data_proxy.go
index 80593bbda74..463dae09a0b 100644
--- a/pkg/setting/setting_data_proxy.go
+++ b/pkg/setting/setting_data_proxy.go
@@ -14,6 +14,7 @@ func readDataProxySettings(iniFile *ini.File, cfg *Cfg) error {
cfg.DataProxyMaxConnsPerHost = dataproxy.Key("max_conns_per_host").MustInt(0)
cfg.DataProxyMaxIdleConns = dataproxy.Key("max_idle_connections").MustInt()
cfg.DataProxyIdleConnTimeout = dataproxy.Key("idle_conn_timeout_seconds").MustInt(90)
+ cfg.ResponseLimit = dataproxy.Key("response_limit").MustInt64(0)
if val, err := dataproxy.Key("max_idle_connections_per_host").Int(); err == nil {
cfg.Logger.Warn("[Deprecated] the configuration setting 'max_idle_connections_per_host' is deprecated, please use 'max_idle_connections' instead")