package models import ( "encoding/json" "errors" "fmt" "math" "net/url" "regexp" "strconv" "strings" "time" "github.com/aws/aws-sdk-go-v2/service/cloudwatch" "github.com/google/uuid" "github.com/grafana/grafana-plugin-sdk-go/backend" "github.com/grafana/grafana-plugin-sdk-go/backend/log" "github.com/grafana/grafana/pkg/tsdb/cloudwatch/kinds/dataquery" ) type ( MetricEditorMode dataquery.MetricEditorMode MetricQueryType dataquery.MetricQueryType GMDApiMode uint32 ) const ( MetricEditorModeBuilder = dataquery.MetricEditorModeBuilder MetricEditorModeRaw = dataquery.MetricEditorModeCode ) const ( MetricQueryTypeSearch = dataquery.MetricQueryTypeSearch MetricQueryTypeQuery = dataquery.MetricQueryTypeInsights ) const ( GMDApiModeMetricStat GMDApiMode = iota GMDApiModeInferredSearchExpression GMDApiModeMathExpression GMDApiModeSQLExpression ) const ( defaultRegion = "default" defaultConsoleURL = "console.aws.amazon.com" usGovConsoleURL = "console.amazonaws-us-gov.com" chinaConsoleURL = "console.amazonaws.cn" ) type SQLExpressionGroupBy struct { Expressions []dataquery.QueryEditorGroupByExpression `json:"expressions"` Type dataquery.QueryEditorArrayExpressionType `json:"type"` } type sqlExpression struct { dataquery.SQLExpression GroupBy *SQLExpressionGroupBy `json:"groupBy,omitempty"` } type CloudWatchQuery struct { logger log.Logger StartTime time.Time EndTime time.Time RefId string Region string Id string Namespace string MetricName string Statistic string Expression string Sql sqlExpression SqlExpression string ReturnData bool Dimensions map[string][]string Period int32 Label string MatchExact bool UsedExpression string TimezoneUTCOffset string MetricQueryType dataquery.MetricQueryType MetricEditorMode dataquery.MetricEditorMode AccountId *string } func (q *CloudWatchQuery) GetGetMetricDataAPIMode() GMDApiMode { if q.MetricQueryType == MetricQueryTypeSearch && q.MetricEditorMode == MetricEditorModeBuilder { if q.IsInferredSearchExpression() { return GMDApiModeInferredSearchExpression } return GMDApiModeMetricStat } else if q.MetricQueryType == MetricQueryTypeSearch && q.MetricEditorMode == MetricEditorModeRaw { return GMDApiModeMathExpression } else if q.MetricQueryType == MetricQueryTypeQuery { return GMDApiModeSQLExpression } q.logger.Warn("Could not resolve CloudWatch metric query type. Falling back to metric stat.", "query", q) return GMDApiModeMetricStat } func (q *CloudWatchQuery) IsMathExpression() bool { return q.MetricQueryType == MetricQueryTypeSearch && q.MetricEditorMode == MetricEditorModeRaw && !q.IsUserDefinedSearchExpression() } func (q *CloudWatchQuery) isSearchExpression() bool { return q.MetricQueryType == MetricQueryTypeSearch && (q.IsUserDefinedSearchExpression() || q.IsInferredSearchExpression()) } func (q *CloudWatchQuery) IsUserDefinedSearchExpression() bool { return q.MetricQueryType == MetricQueryTypeSearch && q.MetricEditorMode == MetricEditorModeRaw && strings.Contains(q.Expression, "SEARCH(") } func (q *CloudWatchQuery) IsInferredSearchExpression() bool { if q.MetricQueryType != MetricQueryTypeSearch || q.MetricEditorMode != MetricEditorModeBuilder { return false } if q.AccountId != nil && *q.AccountId == "all" { return true } if len(q.Dimensions) == 0 { return !q.MatchExact } if !q.MatchExact { return true } for _, values := range q.Dimensions { if len(values) > 1 { return true } for _, v := range values { if v == "*" { return true } } } return false } func (q *CloudWatchQuery) IsMultiValuedDimensionExpression() bool { if q.MetricQueryType != MetricQueryTypeSearch || q.MetricEditorMode != MetricEditorModeBuilder { return false } for _, values := range q.Dimensions { for _, v := range values { if v == "*" { return false } } if len(values) > 1 { return true } } return false } func (q *CloudWatchQuery) BuildDeepLink(startTime time.Time, endTime time.Time) (string, error) { if q.IsMathExpression() || q.MetricQueryType == MetricQueryTypeQuery { return "", nil } link := &cloudWatchLink{ Title: q.RefId, View: "timeSeries", Stacked: false, Region: q.Region, Start: startTime.UTC().Format(time.RFC3339), End: endTime.UTC().Format(time.RFC3339), } if q.isSearchExpression() { metricExpressions := &metricExpression{Expression: q.UsedExpression} metricExpressions.Label = q.Label link.Metrics = []any{metricExpressions} } else { metricStat := []any{q.Namespace, q.MetricName} for dimensionKey, dimensionValues := range q.Dimensions { metricStat = append(metricStat, dimensionKey, dimensionValues[0]) } metricStatMeta := &metricStatMeta{ Stat: q.Statistic, Period: q.Period, } metricStatMeta.Label = q.Label if q.AccountId != nil { metricStatMeta.AccountId = *q.AccountId } metricStat = append(metricStat, metricStatMeta) link.Metrics = []any{metricStat} } linkProps, err := json.Marshal(link) if err != nil { return "", fmt.Errorf("could not marshal link: %w", err) } endpoint, err := getEndpoint(q.Region) if err != nil { return "", err } consoleURL, err := url.Parse(fmt.Sprintf(`https://%s/cloudwatch/deeplink.js`, endpoint)) if err != nil { return "", fmt.Errorf("unable to parse CloudWatch console deep link") } fragment := consoleURL.Query() fragment.Set("graph", string(linkProps)) query := consoleURL.Query() query.Set("region", q.Region) consoleURL.RawQuery = query.Encode() return fmt.Sprintf(`%s#metricsV2:%s`, consoleURL.String(), fragment.Encode()), nil } const timeSeriesQuery = "timeSeriesQuery" var validMetricDataID = regexp.MustCompile(`^[a-z][a-zA-Z0-9_]*$`) type metricsDataQuery struct { dataquery.CloudWatchMetricsQuery Sql *sqlExpression `json:"sql,omitempty"` Type string `json:"type"` TimezoneUTCOffset string `json:"timezoneUTCOffset"` } // ParseMetricDataQueries decodes the metric data queries json, validates, sets default values and returns an array of CloudWatchQueries. // The CloudWatchQuery has a 1 to 1 mapping to a query editor row func ParseMetricDataQueries(dataQueries []backend.DataQuery, startTime time.Time, endTime time.Time, defaultRegion string, logger log.Logger, crossAccountQueryingEnabled bool) ([]*CloudWatchQuery, error) { var metricDataQueries = make(map[string]metricsDataQuery) for _, query := range dataQueries { var metricsDataQuery metricsDataQuery err := json.Unmarshal(query.JSON, &metricsDataQuery) if err != nil { return nil, &QueryError{Err: err, RefID: query.RefID} } queryType := metricsDataQuery.Type if queryType != timeSeriesQuery && queryType != "" { continue } metricDataQueries[query.RefID] = metricsDataQuery } result := make([]*CloudWatchQuery, 0, len(metricDataQueries)) for refId, mdq := range metricDataQueries { cwQuery := &CloudWatchQuery{ logger: logger, StartTime: startTime, EndTime: endTime, RefId: refId, Id: mdq.Id, Region: mdq.Region, Namespace: mdq.Namespace, TimezoneUTCOffset: mdq.TimezoneUTCOffset, } if mdq.MetricName != nil { cwQuery.MetricName = *mdq.MetricName } if mdq.MetricQueryType != nil { cwQuery.MetricQueryType = *mdq.MetricQueryType } if mdq.Sql != nil { cwQuery.Sql = *mdq.Sql } if mdq.SqlExpression != nil { cwQuery.SqlExpression = *mdq.SqlExpression } if mdq.Expression != nil { cwQuery.Expression = *mdq.Expression } if mdq.Label != nil { cwQuery.Label = *mdq.Label } if err := cwQuery.validateAndSetDefaults(refId, mdq, startTime, endTime, defaultRegion, crossAccountQueryingEnabled); err != nil { return nil, &QueryError{Err: err, RefID: refId} } cwQuery.applyMacros(startTime, endTime) cwQuery.migrateLegacyQuery(mdq) result = append(result, cwQuery) } return result, nil } func (q *CloudWatchQuery) applyMacros(startTime, endTime time.Time) { if q.GetGetMetricDataAPIMode() == GMDApiModeMathExpression { q.Expression = strings.ReplaceAll(q.Expression, "$__period_auto", strconv.Itoa(calculatePeriodBasedOnTimeRange(startTime, endTime))) } } func (q *CloudWatchQuery) migrateLegacyQuery(query metricsDataQuery) { q.Statistic = getStatistic(query) q.Label = getLabel(query) } func (q *CloudWatchQuery) validateAndSetDefaults(refId string, metricsDataQuery metricsDataQuery, startTime, endTime time.Time, defaultRegionValue string, crossAccountQueryingEnabled bool) error { if metricsDataQuery.Statistic == nil && metricsDataQuery.Statistics == nil { return backend.DownstreamError(fmt.Errorf("query must have either statistic or statistics field")) } var err error parsedPeriod, err := getPeriod(metricsDataQuery, startTime, endTime) if err != nil { return err } if parsedPeriod < math.MinInt32 || parsedPeriod > math.MaxInt32 { return fmt.Errorf("query period doesn't fit int32: %d", parsedPeriod) } q.Period = int32(parsedPeriod) q.Dimensions = map[string][]string{} if metricsDataQuery.Dimensions != nil { q.Dimensions, err = parseDimensions(*metricsDataQuery.Dimensions) if err != nil { return fmt.Errorf("failed to parse dimensions: %v", err) } } if crossAccountQueryingEnabled { q.AccountId = metricsDataQuery.AccountId } if metricsDataQuery.Id == "" { // Why not just use refId if id is not specified in the frontend? When specifying an id in the editor, // and alphabetical must be used. The id must be unique, so if an id like for example a, b or c would be used, // it would likely collide with some ref id. That's why the `query` prefix is used. suffix := refId if !validMetricDataID.MatchString(suffix) { newUUID := uuid.NewString() suffix = strings.ReplaceAll(newUUID, "-", "") } q.Id = fmt.Sprintf("query%s", suffix) } q.MatchExact = true if metricsDataQuery.MatchExact != nil { q.MatchExact = *metricsDataQuery.MatchExact } q.ReturnData = true if metricsDataQuery.Hide != nil { q.ReturnData = !*metricsDataQuery.Hide } if metricsDataQuery.Type == "" { // If no type is provided we assume we are called by alerting service, which requires to return data! // Note, this is sort of a hack, but the official Grafana interfaces do not carry the information // who (which service) called the TsdbQueryEndpoint.Query(...) function. q.ReturnData = true } if metricsDataQuery.MetricEditorMode == nil && metricsDataQuery.Expression != nil && len(*metricsDataQuery.Expression) > 0 { // this should only ever happen if this is an alerting query that has not yet been migrated in the frontend q.MetricEditorMode = MetricEditorModeRaw } else { if metricsDataQuery.MetricEditorMode != nil { q.MetricEditorMode = *metricsDataQuery.MetricEditorMode } else { q.MetricEditorMode = MetricEditorModeBuilder } } if q.Region == defaultRegion { q.Region = defaultRegionValue } return nil } // getStatistic determines the value of Statistic in a CloudWatchQuery from the metricsDataQuery input // migrates queries that has a `statistics` field to use the `statistic` field instead. // In case the query used more than one stat, the first stat in the slice will be used in the statistic field // Read more here https://github.com/grafana/grafana/issues/30629 func getStatistic(query metricsDataQuery) string { // If there's not a statistic property in the json, we know it's the legacy format and then it has to be migrated if query.Statistic == nil { if len(query.Statistics) > 0 { return query.Statistics[0] } // if there isn't a statistic property in the legacy format fall back to Average return "Average" } return *query.Statistic } var aliasPatterns = map[string]string{ "metric": `${PROP('MetricName')}`, "namespace": `${PROP('Namespace')}`, "period": `${PROP('Period')}`, "region": `${PROP('Region')}`, "stat": `${PROP('Stat')}`, "label": `${LABEL}`, } var legacyAliasRegexp = regexp.MustCompile(`{{\s*(.+?)\s*}}`) func getLabel(query metricsDataQuery) string { deprecatedAlias := query.Alias //nolint:staticcheck if query.Label != nil { return *query.Label } if deprecatedAlias != nil && *deprecatedAlias == "" { return "" } var result string fullAliasField := "" if deprecatedAlias != nil { fullAliasField = *deprecatedAlias } matches := legacyAliasRegexp.FindAllStringSubmatch(fullAliasField, -1) for _, groups := range matches { fullMatch := groups[0] subgroup := groups[1] if dynamicLabel, ok := aliasPatterns[subgroup]; ok { fullAliasField = strings.ReplaceAll(fullAliasField, fullMatch, dynamicLabel) } else { fullAliasField = strings.ReplaceAll(fullAliasField, fullMatch, fmt.Sprintf(`${PROP('Dim.%s')}`, subgroup)) } } result = fullAliasField return result } func calculatePeriodBasedOnTimeRange(startTime, endTime time.Time) int { deltaInSeconds := endTime.Sub(startTime).Seconds() periods := getRetainedPeriods(time.Since(startTime)) datapoints := int(math.Ceil(deltaInSeconds / 2000)) period := periods[len(periods)-1] for _, value := range periods { if datapoints <= value { period = value break } } return period } func getPeriod(query metricsDataQuery, startTime, endTime time.Time) (int, error) { periodString := "" if query.Period != nil { periodString = *query.Period } var period int var err error if strings.ToLower(periodString) == "auto" || periodString == "" { period = calculatePeriodBasedOnTimeRange(startTime, endTime) } else { period, err = strconv.Atoi(periodString) if err != nil { d, err := time.ParseDuration(periodString) if err != nil { return 0, fmt.Errorf("failed to parse period as duration: %v", err) } period = int(d.Seconds()) } } return period, nil } func getRetainedPeriods(timeSince time.Duration) []int { // See https://aws.amazon.com/about-aws/whats-new/2016/11/cloudwatch-extends-metrics-retention-and-new-user-interface/ if timeSince > time.Duration(455)*24*time.Hour { return []int{21600, 86400} } else if timeSince > time.Duration(63)*24*time.Hour { return []int{3600, 21600, 86400} } else if timeSince > time.Duration(15)*24*time.Hour { return []int{300, 900, 3600, 21600, 86400} } else { return []int{60, 300, 900, 3600, 21600, 86400} } } func parseDimensions(dimensions dataquery.Dimensions) (map[string][]string, error) { parsedDimensions := make(map[string][]string) for k, v := range dimensions { // This is for backwards compatibility. Before 6.5 dimensions values were stored as strings and not arrays if v.String != nil { parsedDimensions[k] = []string{*v.String} } else if len(v.ArrayOfString) > 0 { parsedDimensions[k] = append(parsedDimensions[k], v.ArrayOfString...) } else { return nil, errors.New("unknown type as dimension value") } } return parsedDimensions, nil } func getEndpoint(region string) (string, error) { resolver := cloudwatch.NewDefaultEndpointResolver() endpoint, err := resolver.ResolveEndpoint(region, cloudwatch.EndpointResolverOptions{}) if err != nil { return "", fmt.Errorf("resolve endpoint failed: %w", err) } consoleURL := defaultConsoleURL switch endpoint.PartitionID { case "aws-us-gov": consoleURL = usGovConsoleURL case "aws-cn": consoleURL = chinaConsoleURL } return fmt.Sprintf("%s.%s", region, consoleURL), nil }