Alerting: Improve Prometheus Alert Rule error message (#29390)

* provide better ctx for prom err

* rework

* Update pkg/tsdb/prometheus/prometheus.go

Co-authored-by: Arve Knudsen <arve.knudsen@gmail.com>

* Update pkg/tsdb/prometheus/prometheus.go

Co-authored-by: Arve Knudsen <arve.knudsen@gmail.com>

* call func

Co-authored-by: Arve Knudsen <arve.knudsen@gmail.com>

pkg/services/alerting/conditions/query.go

@@ -6,6 +6,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/grafana/grafana/pkg/tsdb/prometheus"
+
 	gocontext "context"
 
 	"github.com/grafana/grafana-plugin-sdk-go/data"
@@ -159,11 +161,7 @@ func (c *QueryCondition) executeQuery(context *alerting.EvalContext, timeRange *
 
 	resp, err := c.HandleRequest(context.Ctx, getDsInfo.Result, req)
 	if err != nil {
-		if errors.Is(err, gocontext.DeadlineExceeded) {
+		return nil, toCustomError(err)
-			return nil, fmt.Errorf("alert execution exceeded the timeout")
-		}
-
-		return nil, fmt.Errorf("tsdb.HandleRequest() error %v", err)
 	}
 
 	for _, v := range resp.Results {
@@ -360,3 +358,18 @@ func FrameToSeriesSlice(frame *data.Frame) (tsdb.TimeSeriesSlice, error) {
 
 	return seriesSlice, nil
 }
+
+func toCustomError(err error) error {
+	// is context timeout
+	if errors.Is(err, gocontext.DeadlineExceeded) {
+		return fmt.Errorf("alert execution exceeded the timeout")
+	}
+
+	// is Prometheus error
+	if prometheus.IsAPIError(err) {
+		return prometheus.ConvertAPIError(err)
+	}
+
+	// generic fallback
+	return fmt.Errorf("tsdb.HandleRequest() error %v", err)
+}

pkg/tsdb/prometheus/prometheus.go

@@ -2,6 +2,7 @@ package prometheus
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"regexp"
 	"strings"
@@ -15,7 +16,7 @@ import (
 	"github.com/grafana/grafana/pkg/infra/log"
 	"github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/tsdb"
-	api "github.com/prometheus/client_golang/api"
+	"github.com/prometheus/client_golang/api"
 	apiv1 "github.com/prometheus/client_golang/api/prometheus/v1"
 	"github.com/prometheus/common/model"
 )
@@ -216,3 +217,18 @@ func parseResponse(value model.Value, query *PrometheusQuery) (*tsdb.QueryResult
 
 	return queryRes, nil
 }
+
+func IsAPIError(err error) bool {
+	// Have to use errors.As to compare Prometheus errors, since errors.Is won't work due to Prometheus
+	// errors being pointers and errors.Is ends up comparing them by pointer address
+	var e *apiv1.Error
+	return errors.As(err, &e)
+}
+
+func ConvertAPIError(err error) error {
+	var e *apiv1.Error
+	if errors.As(err, &e) {
+		return fmt.Errorf("%s: %s", e.Msg, e.Detail)
+	}
+	return err
+}