Alerting: Recording rules understands errors embedded in dataframes (#88946)

* Make MakeDependencyError public for tests in another package

* Create tests for errors in eval results

* Extract logic to pull frame errors out into exported function

* Maybe we can drop cyclomatic complexity lint suppression now?

* extract frame errors and fail recording rules if frames contain error

* Fix up retry logic to actually work

* Do not retry non retryable errors

pkg/expr/errors.go

@@ -63,7 +63,7 @@ var DependencyError = errutil.NewBase(
 	depErrStr,
 	errutil.WithPublic(depErrStr))
 
-func makeDependencyError(refID, depRefID string) error {
+func MakeDependencyError(refID, depRefID string) error {
 	data := errutil.TemplateData{
 		Public: map[string]interface{}{
 			"refId":    refID,

pkg/expr/graph.go

@@ -90,7 +90,7 @@ func (dp *DataPipeline) execute(c context.Context, now time.Time, s *Service) (m
 			if res, ok := vars[neededVar]; ok {
 				if res.Error != nil {
 					errResult := mathexp.Results{
-						Error: makeDependencyError(node.RefID(), neededVar),
+						Error: MakeDependencyError(node.RefID(), neededVar),
 					}
 					vars[node.RefID()] = errResult
 					hasDepError = true