Data sources: Improve error messages for grpc errors (#102372)

* Data sources: Improve error messages for grpc errors

* Improve code comments

* Fix lint

* Update connection issue message

* Update name

* Update comment

* Update, rename and add test

* Update, remove POC change

* Fix lint

pkg/plugins/backendplugin/grpcplugin/client_v2.go

@@ -172,6 +172,14 @@ func (c *ClientV2) QueryData(ctx context.Context, req *backend.QueryDataRequest)
 			return nil, plugins.ErrMethodNotImplemented
 		}
 
+		if status.Code(err) == codes.Unavailable {
+			return nil, plugins.ErrPluginGrpcConnectionUnavailableBase.Errorf("%v", err)
+		}
+
+		if status.Code(err) == codes.ResourceExhausted {
+			return nil, plugins.ErrPluginGrpcResourceExhaustedBase.Errorf("%v", err)
+		}
+
 		if errorSource, ok := backend.ErrorSourceFromGrpcStatusError(ctx, err); ok {
 			return nil, handleGrpcStatusError(ctx, errorSource, err)
 		}

pkg/plugins/errors.go

@@ -35,4 +35,16 @@ var (
 	// Exposed as a base error to wrap it with plugin cancelled errors.
 	ErrPluginRequestCanceledErrorBase = errutil.ClientClosedRequest("plugin.requestCanceled",
 		errutil.WithPublicMessage("Plugin request canceled"))
+
+	// ErrPluginGrpcResourceExhaustedBase error returned when a plugin response is larger than the grpc limit.
+	// Exposed as a base error to wrap it with plugin resource exhausted errors.
+	ErrPluginGrpcResourceExhaustedBase = errutil.Internal("plugin.resourceExhausted",
+		errutil.WithPublicMessage("The response is too large. Please try to reduce the time range or narrow down your query to return fewer data points."),
+		errutil.WithDownstream())
+
+	// ErrPluginGrpcConnectionUnavailableBase error returned when a plugin connection issue occurs.
+	// Exposed as a base error to wrap it with plugin connection issue errors.
+	ErrPluginGrpcConnectionUnavailableBase = errutil.Internal("plugin.connectionUnavailable",
+		errutil.WithPublicMessage("Data source became unavailable during request. Please try again."),
+		errutil.WithDownstream())
 )

pkg/plugins/manager/client/client.go

@@ -30,6 +30,14 @@ var (
 	errNilSender  = errors.New("sender cannot be nil")
 )
 
+// passthroughErrors contains a list of errors that should be returned directly to the caller without wrapping
+var passthroughErrors = []error{
+	plugins.ErrPluginUnavailable,
+	plugins.ErrMethodNotImplemented,
+	plugins.ErrPluginGrpcResourceExhaustedBase,
+	plugins.ErrPluginGrpcConnectionUnavailableBase,
+}
+
 type Service struct {
 	pluginRegistry registry.Service
 }
@@ -52,12 +60,10 @@ func (s *Service) QueryData(ctx context.Context, req *backend.QueryDataRequest)
 
 	resp, err := p.QueryData(ctx, req)
 	if err != nil {
-		if errors.Is(err, plugins.ErrMethodNotImplemented) {
+		for _, e := range passthroughErrors {
-			return nil, err
+			if errors.Is(err, e) {
-		}
+				return nil, err
-
+			}
-		if errors.Is(err, plugins.ErrPluginUnavailable) {
-			return nil, err
 		}
 
 		if errors.Is(err, context.Canceled) {

pkg/plugins/manager/client/client_test.go

@@ -25,24 +25,39 @@ func TestQueryData(t *testing.T) {
 
 	t.Run("Non-empty registry", func(t *testing.T) {
 		tcs := []struct {
-			err           error
+			err               error
-			expectedError error
+			expectedError     error
+			shouldPassThrough bool
 		}{
 			{
-				err:           plugins.ErrPluginUnavailable,
+				err:               plugins.ErrPluginUnavailable,
-				expectedError: plugins.ErrPluginUnavailable,
+				expectedError:     plugins.ErrPluginUnavailable,
+				shouldPassThrough: true,
 			},
 			{
-				err:           plugins.ErrMethodNotImplemented,
+				err:               plugins.ErrMethodNotImplemented,
-				expectedError: plugins.ErrMethodNotImplemented,
+				expectedError:     plugins.ErrMethodNotImplemented,
+				shouldPassThrough: true,
 			},
 			{
-				err:           errors.New("surprise surprise"),
+				err:               errors.New("surprise surprise"),
-				expectedError: plugins.ErrPluginRequestFailureErrorBase,
+				expectedError:     plugins.ErrPluginRequestFailureErrorBase,
+				shouldPassThrough: false,
 			},
 			{
-				err:           context.Canceled,
+				err:               context.Canceled,
-				expectedError: plugins.ErrPluginRequestCanceledErrorBase,
+				expectedError:     plugins.ErrPluginRequestCanceledErrorBase,
+				shouldPassThrough: false,
+			},
+			{
+				err:               plugins.ErrPluginGrpcConnectionUnavailableBase.Errorf("unavailable"),
+				expectedError:     plugins.ErrPluginGrpcConnectionUnavailableBase.Errorf("unavailable"),
+				shouldPassThrough: true,
+			},
+			{
+				err:               plugins.ErrPluginGrpcResourceExhaustedBase.Errorf("exhausted"),
+				expectedError:     plugins.ErrPluginGrpcResourceExhaustedBase.Errorf("exhausted"),
+				shouldPassThrough: true,
 			},
 		}
 
@@ -69,7 +84,11 @@ func TestQueryData(t *testing.T) {
 					},
 				})
 				require.Error(t, err)
-				require.ErrorIs(t, err, tc.expectedError)
+				if tc.shouldPassThrough {
+					require.Equal(t, tc.err, err)
+				} else {
+					require.ErrorIs(t, err, tc.expectedError)
+				}
 			})
 		}
 	})