Azure: Load custom clouds from ini file (#87667)

* Load custom clouds from config file

* Update docs

* Use the correct list of clouds, add test, fix error condition handling

* Remove on custom cloud from sample.ini and docs

* Remove unnecessary else block

* Use cached json instead of serializing with each request

* Update grafana-azure-sdk-go version to v2.0.4

* update configure-grafana entry for clouds_config

* fix lint errors

* fix lint errors

---------

Co-authored-by: Jeremy Angel (from Dev Box) <jeremyangel@microsoft.com>

conf/defaults.ini

@@ -921,6 +921,10 @@ forward_settings_to_plugins = cloudwatch, grafana-athena-datasource, grafana-red
 # Default value is AzureCloud (i.e. public cloud)
 cloud = AzureCloud
 
+# A customized list of Azure cloud settings and properties, used by data sources which need this information when run in non-standard azure environments
+# When specified, this list will replace the default cloud list of AzureCloud, AzureChinaCloud, AzureUSGovernment and AzureGermanCloud
+clouds_config = 
+
 # Specifies whether Grafana hosted in Azure service with Managed Identity configured (e.g. Azure Virtual Machines instance)
 # If enabled, the managed identity can be used for authentication of Grafana in Azure services
 # Disabled by default, needs to be explicitly enabled

conf/sample.ini

@@ -839,6 +839,22 @@
 # Default value is AzureCloud (i.e. public cloud)
 ;cloud = AzureCloud
 
+# A customized list of Azure cloud settings and properties, used by data sources which need this information when run in non-standard azure environments
+# When specified, this list will replace the default cloud list of AzureCloud, AzureChinaCloud, AzureUSGovernment and AzureGermanCloud
+;clouds_config = `[
+;		{
+;			"name":"CustomCloud1",
+;			"displayName":"Custom Cloud 1",
+;			"aadAuthority":"https://login.cloud1.contoso.com/",
+;			"properties":{
+;				"azureDataExplorerSuffix": ".kusto.windows.cloud1.contoso.com",
+;				"logAnalytics":            "https://api.loganalytics.cloud1.contoso.com",
+;				"portal":                  "https://portal.azure.cloud1.contoso.com",
+;				"prometheusResourceId":    "https://prometheus.monitor.azure.cloud1.contoso.com",
+;				"resourceManager":         "https://management.azure.cloud1.contoso.com"
+;			}
+;		}]`
+
 # Specifies whether Grafana hosted in Azure service with Managed Identity configured (e.g. Azure Virtual Machines instance)
 # If enabled, the managed identity can be used for authentication of Grafana in Azure services
 # Disabled by default, needs to be explicitly enabled

docs/sources/setup-grafana/configure-grafana/_index.md

@@ -1175,6 +1175,28 @@ Azure cloud environment where Grafana is hosted:
 | US Government cloud                              | AzureUSGovernment      |
 | Microsoft German national cloud ("Black Forest") | AzureGermanCloud       |
 
+### clouds_config
+
+The JSON config defines a list of Azure clouds and their associated properties when hosted in custom Azure environments.
+
+For example:
+
+```ini
+clouds_config = `[
+		{
+			"name":"CustomCloud1",
+			"displayName":"Custom Cloud 1",
+			"aadAuthority":"https://login.cloud1.contoso.com/",
+			"properties":{
+				"azureDataExplorerSuffix": ".kusto.windows.cloud1.contoso.com",
+				"logAnalytics":            "https://api.loganalytics.cloud1.contoso.com",
+				"portal":                  "https://portal.azure.cloud1.contoso.com",
+				"prometheusResourceId":    "https://prometheus.monitor.azure.cloud1.contoso.com",
+				"resourceManager":         "https://management.azure.cloud1.contoso.com"
+			}
+		}]`
+```
+
 ### managed_identity_enabled
 
 Specifies whether Grafana hosted in Azure service with Managed Identity configured (e.g. Azure Virtual Machines instance). Disabled by default, needs to be explicitly enabled.

go.mod

@@ -96,7 +96,7 @@ require (
 	github.com/grafana/gofpdf v0.0.0-20231002120153-857cc45be447 // @grafana/sharing-squad
 	github.com/grafana/gomemcache v0.0.0-20231023152154-6947259a0586 // @grafana/grafana-operator-experience-squad
 	github.com/grafana/grafana-aws-sdk v0.25.1 // @grafana/aws-datasources
-	github.com/grafana/grafana-azure-sdk-go/v2 v2.0.3 // @grafana/partner-datasources
+	github.com/grafana/grafana-azure-sdk-go/v2 v2.0.4 // @grafana/partner-datasources
 	github.com/grafana/grafana-google-sdk-go v0.1.0 // @grafana/partner-datasources
 	github.com/grafana/grafana-openapi-client-go v0.0.0-20231213163343-bd475d63fb79 // @grafana/grafana-backend-group
 	github.com/grafana/grafana-plugin-sdk-go v0.232.0 // @grafana/plugins-platform-backend

go.sum

@@ -2172,8 +2172,8 @@ github.com/grafana/gomemcache v0.0.0-20231023152154-6947259a0586 h1:/of8Z8taCPft
 github.com/grafana/gomemcache v0.0.0-20231023152154-6947259a0586/go.mod h1:PGk3RjYHpxMM8HFPhKKo+vve3DdlPUELZLSDEFehPuU=
 github.com/grafana/grafana-aws-sdk v0.25.1 h1:waJERJueqB1GldclAh5+tBcY9Ju9AOO9xYSJEnOAuf4=
 github.com/grafana/grafana-aws-sdk v0.25.1/go.mod h1:3zghFF6edrxn0d6k6X9HpGZXDH+VfA+MwD2Pc/9X0ec=
-github.com/grafana/grafana-azure-sdk-go/v2 v2.0.3 h1:5qW9WVDpAxJx7db3Ir2BsHDetLhAFCwrAFlZjzCqTDE=
+github.com/grafana/grafana-azure-sdk-go/v2 v2.0.4 h1:z6amQ286IJSBctHf6c+ibJq/v0+TvmEjVkrdMNBd4uY=
-github.com/grafana/grafana-azure-sdk-go/v2 v2.0.3/go.mod h1:s8GLONgVh/svnSsO0Eo+OgXc/RZqozI5/0n+pNm3MEE=
+github.com/grafana/grafana-azure-sdk-go/v2 v2.0.4/go.mod h1:aKlFPE36IDa8qccRg3KbgZX3MQ5xymS3RelT4j6kkVU=
 github.com/grafana/grafana-google-sdk-go v0.1.0 h1:LKGY8z2DSxKjYfr2flZsWgTRTZ6HGQbTqewE3JvRaNA=
 github.com/grafana/grafana-google-sdk-go v0.1.0/go.mod h1:Vo2TKWfDVmNTELBUM+3lkrZvFtBws0qSZdXhQxRdJrE=
 github.com/grafana/grafana-openapi-client-go v0.0.0-20231213163343-bd475d63fb79 h1:r+mU5bGMzcXCRVAuOrTn54S80qbfVkvTdUJZfSfTNbs=

go.work.sum

@@ -659,6 +659,7 @@ github.com/grafana/grafana-plugin-sdk-go v0.227.1-0.20240426134450-5fe9f7b9dfd4/
 github.com/grafana/grafana-plugin-sdk-go v0.227.1-0.20240430073540-ce4d126ae8b8 h1:pyWJN79uW8QHZiQRasHGLCEkXSr3k6HCjdr0J2jZ3rU=
 github.com/grafana/grafana-plugin-sdk-go v0.227.1-0.20240430073540-ce4d126ae8b8/go.mod h1:u4K9vVN6eU86loO68977eTXGypC4brUCnk4sfDzutZU=
 github.com/grafana/grafana-plugin-sdk-go v0.228.0/go.mod h1:u4K9vVN6eU86loO68977eTXGypC4brUCnk4sfDzutZU=
+github.com/grafana/grafana-plugin-sdk-go v0.230.0 h1:Y4IL+eT1jXqTCctlNzdCvxAozpBZ8xEsRGWjGAVRXxo=
 github.com/grafana/grafana-plugin-sdk-go v0.229.0/go.mod h1:6V6ikT4ryva8MrAp7Bdz5fTJx3/ztzKvpMJFfpzr4CI=
 github.com/grafana/grafana-plugin-sdk-go v0.230.0/go.mod h1:6V6ikT4ryva8MrAp7Bdz5fTJx3/ztzKvpMJFfpzr4CI=
 github.com/grafana/grafana-plugin-sdk-go v0.231.1-0.20240523124942-62dae9836284/go.mod h1:bNgmNmub1I7Mc8dzIncgNqHC5jTgSZPPHlZ3aG8HKJQ=

pkg/services/pluginsintegration/pluginconfig/request.go

@@ -95,6 +95,10 @@ func (s *RequestConfigProvider) PluginRequestConfig(ctx context.Context, pluginI
 			m[azsettings.AzureCloud] = azureSettings.Cloud
 		}
 
+		if len(azureSettings.CustomCloudListJSON) > 0 {
+			m[azsettings.AzureCustomCloudsConfig] = azureSettings.CustomCloudListJSON
+		}
+
 		if azureSettings.ManagedIdentityEnabled {
 			m[azsettings.ManagedIdentityEnabled] = "true"
 

pkg/services/pluginsintegration/pluginconfig/request_test.go

@@ -335,6 +335,38 @@ func TestRequestConfigProvider_PluginRequestConfig_azure(t *testing.T) {
 		})
 	})
 
+	t.Run("azure settings include custom clouds when set", func(t *testing.T) {
+		cfg := setting.NewCfg()
+		cfg.Azure = azSettings
+
+		customCloudJson := `[{"name":"CustomCloud1","displayName":"Custom Cloud 1","aadAuthority":"https://login.contoso.com/","properties":null}]`
+		err := azSettings.SetCustomClouds(customCloudJson)
+		require.NoError(t, err)
+
+		// Sanity check to make sure SetCustomClouds call above also desirializes the JSON
+		require.Equal(t, len(azSettings.CustomCloudList), 1)
+
+		pCfg, err := ProvidePluginInstanceConfig(cfg, setting.ProvideProvider(cfg), featuremgmt.WithFeatures())
+		require.NoError(t, err)
+
+		p := NewRequestConfigProvider(pCfg)
+		require.Subset(t, p.PluginRequestConfig(context.Background(), "grafana-azure-monitor-datasource", nil), map[string]string{
+			"GFAZPL_AZURE_CLOUD": "AzureCloud", "GFAZPL_MANAGED_IDENTITY_ENABLED": "true",
+			"GFAZPL_MANAGED_IDENTITY_CLIENT_ID":                         "mock_managed_identity_client_id",
+			"GFAZPL_AZURE_CLOUDS_CONFIG":                                customCloudJson,
+			"GFAZPL_WORKLOAD_IDENTITY_ENABLED":                          "true",
+			"GFAZPL_WORKLOAD_IDENTITY_TENANT_ID":                        "mock_workload_identity_tenant_id",
+			"GFAZPL_WORKLOAD_IDENTITY_CLIENT_ID":                        "mock_workload_identity_client_id",
+			"GFAZPL_WORKLOAD_IDENTITY_TOKEN_FILE":                       "mock_workload_identity_token_file",
+			"GFAZPL_USER_IDENTITY_ENABLED":                              "true",
+			"GFAZPL_USER_IDENTITY_FALLBACK_SERVICE_CREDENTIALS_ENABLED": "true",
+			"GFAZPL_USER_IDENTITY_TOKEN_URL":                            "mock_user_identity_token_url",
+			"GFAZPL_USER_IDENTITY_CLIENT_ID":                            "mock_user_identity_client_id",
+			"GFAZPL_USER_IDENTITY_CLIENT_SECRET":                        "mock_user_identity_client_secret",
+			"GFAZPL_USER_IDENTITY_ASSERTION":                            "username",
+		})
+	})
+
 	t.Run("does not use the azure settings for a non-Azure plugin", func(t *testing.T) {
 		cfg := setting.NewCfg()
 		cfg.Azure = azSettings

pkg/setting/setting_azure.go

@@ -72,6 +72,12 @@ func (cfg *Cfg) readAzureSettings() {
 		azureSettings.UserIdentityFallbackCredentialsEnabled = azureSection.Key("user_identity_fallback_credentials_enabled").MustBool(true)
 	}
 
+	if customCloudsJSON := azureSection.Key("clouds_config").MustString(""); customCloudsJSON != "" {
+		if err := azureSettings.SetCustomClouds(customCloudsJSON); err != nil {
+			cfg.Logger.Error("Failed to parse custom Azure cloud settings", "err", err.Error())
+		}
+	}
+
 	azureSettings.ForwardSettingsPlugins = util.SplitString(azureSection.Key("forward_settings_to_plugins").String())
 
 	cfg.Azure = azureSettings