frontend for trim/apply defaults and some bug fixing (#33561)

* remove empty object and workaround on list

* frontend

* add toggle on frontend

cue/data/gen.cue

@@ -10,7 +10,7 @@ Family: scuemata.#Family & {
                 // TODO must isolate or remove identifiers local to a Grafana instance...?
                 id?: number
                 // Unique dashboard identifier that can be generated by anyone. string (8-40)
-                uid: string
+                uid?: string
                 // Title of dashboard.
                 title?: string
                 // Description of dashboard.
@@ -181,7 +181,7 @@ Family: scuemata.#Family & {
                             //   nullValueMode?: NullValueMode;
 
                             //   // The behavior when clicking on a result
-                            //   links?: DataLink[];
+                            links?: [...]
 
                             // Alternative to empty string
                             noValue?: string

packages/grafana-data/src/types/config.ts

@@ -46,6 +46,7 @@ export interface FeatureToggles {
 
   live: boolean;
   ngalert: boolean;
+  trimDefaults: boolean;
   panelLibrary: boolean;
   accesscontrol: boolean;
 

packages/grafana-runtime/src/config.ts

@@ -59,6 +59,7 @@ export class GrafanaBootConfig implements GrafanaConfig {
     panelLibrary: false,
     reportVariables: false,
     accesscontrol: false,
+    trimDefaults: false,
   };
   licenseInfo: LicenseInfo = {} as LicenseInfo;
   rendererAvailable = false;

pkg/api/plugins.go

@@ -206,10 +206,19 @@ func (hs *HTTPServer) GetPluginMarkdown(c *models.ReqContext) response.Response
 }
 
 func (hs *HTTPServer) ImportDashboard(c *models.ReqContext, apiCmd dtos.ImportDashboardCommand) response.Response {
+	var err error
 	if apiCmd.PluginId == "" && apiCmd.Dashboard == nil {
 		return response.Error(422, "Dashboard must be set", nil)
 	}
 
+	trimDefaults := c.QueryBoolWithDefault("trimdefaults", true)
+	if trimDefaults && !hs.LoadSchemaService.IsDisabled() {
+		apiCmd.Dashboard, err = hs.LoadSchemaService.DashboardApplyDefaults(apiCmd.Dashboard)
+		if err != nil {
+			return response.Error(500, "Error while applying default value to the dashboard json", err)
+		}
+	}
+
 	dashInfo, err := hs.PluginManager.ImportDashboard(apiCmd.PluginId, apiCmd.Path, c.OrgId, apiCmd.FolderId,
 		apiCmd.Dashboard, apiCmd.Overwrite, apiCmd.Inputs, c.SignedInUser, hs.DataService)
 	if err != nil {

pkg/schema/load/applydefault_test.go

@@ -51,7 +51,6 @@ func TestGenerate(t *testing.T) {
 		})
 	}
 
-	t.Skip()
 	for _, c := range cases {
 		t.Run(c.Name+" trim default value", func(t *testing.T) {
 			var r cue.Runtime

pkg/schema/load/generic.go

@@ -3,6 +3,7 @@ package load
 import (
 	"bytes"
 	"fmt"
+	"strings"
 
 	"cuelang.org/go/cue"
 	"cuelang.org/go/cue/load"
@@ -149,6 +150,7 @@ func (gvs *genericVersionedSchema) TrimDefaults(r schema.Resource) (schema.Resou
 		return r, err
 	}
 	re, err := convertCUEValueToString(rv)
+	fmt.Println("the trimed fields would be: ", re)
 	if err != nil {
 		return r, err
 	}
@@ -171,9 +173,10 @@ func removeDefaultHelper(inputdef cue.Value, input cue.Value) (cue.Value, bool,
 		if err != nil {
 			return rv, false, err
 		}
+		keySet := make(map[string]bool)
 		for iter.Next() {
 			lable, _ := iter.Value().Label()
-
+			keySet[lable] = true
 			lv := input.LookupPath(cue.MakePath(cue.Str(lable)))
 			if err != nil {
 				continue
@@ -185,6 +188,17 @@ func removeDefaultHelper(inputdef cue.Value, input cue.Value) (cue.Value, bool,
 				}
 			}
 		}
+		// Get all the fields that are not defined in schema yet for panel
+		iter, err = input.Fields()
+		if err != nil {
+			return rv, false, err
+		}
+		for iter.Next() {
+			lable, _ := iter.Value().Label()
+			if exists := keySet[lable]; !exists {
+				rv = rv.FillPath(cue.MakePath(cue.Str(lable)), iter.Value())
+			}
+		}
 		return rv, false, nil
 	case cue.ListKind:
 		val, _ := inputdef.Default()
@@ -194,7 +208,6 @@ func removeDefaultHelper(inputdef cue.Value, input cue.Value) (cue.Value, bool,
 			return rv, true, nil
 		}
 		ele := inputdef.LookupPath(cue.MakePath(cue.AnyIndex))
-		fmt.Println("xxxxxxxxxxxxxxxxxxxxx ", ele.IncompleteKind())
 		if ele.IncompleteKind() == cue.BottomKind {
 			return rv, true, nil
 		}
@@ -203,17 +216,23 @@ func removeDefaultHelper(inputdef cue.Value, input cue.Value) (cue.Value, bool,
 		if err != nil {
 			return rv, true, nil
 		}
-		index := 0
+		var iterlist []string
 		for iter.Next() {
 			re, isEqual, err := removeDefaultHelper(ele, iter.Value())
 			if err == nil && !isEqual {
-				rv = rv.FillPath(cue.MakePath(cue.Index(index)), re)
+				reString, err := convertCUEValueToString(re)
-				index++
+				if err != nil {
+					return rv, true, nil
+				}
+				iterlist = append(iterlist, reString)
 			}
 		}
-
+		iterlistContent := fmt.Sprintf("[%s]", strings.Join(iterlist, ","))
-		// rv = rv.FillPath(cue.MakePath(cue.Str(lable)), rv)
+		liInstance, err := rt.Compile("resource", []byte(iterlistContent))
-		return rv, false, nil
+		if err != nil {
+			return rv, false, err
+		}
+		return liInstance.Value(), false, nil
 	default:
 		val, _ := inputdef.Default()
 		err1 := input.Subsume(val)

pkg/schema/load/testdata/artifacts/dashboards/trimdefault/test4

@@ -0,0 +1,73 @@
+Verifies common usecases for trimdefault/applydefault functions:
+* open structure should be kept when fields not present
+
+-- CUE --
+{ 
+  templating?: list: [...{...}]
+}
+
+-- Full --
+{
+  "templating": {
+    "list": [
+      {
+        "allValue": null,
+        "current": {
+            "text": "America",
+            "value": "America"
+        },
+        "datasource": "gdev-postgres",
+        "definition": "",
+        "hide": 0,
+        "includeAll": false,
+        "label": "Datacenter",
+        "multi": false,
+        "name": "datacenter",
+        "options": [],
+        "query": "SELECT DISTINCT datacenter FROM grafana_metric",
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 1,
+        "tagValuesQuery": "",
+        "tags": [],
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      }
+    ]
+  }
+}
+
+-- Trimed --
+{
+  "templating": {
+    "list": [
+      {
+        "allValue": null,
+        "current": {
+            "text": "America",
+            "value": "America"
+        },
+        "datasource": "gdev-postgres",
+        "definition": "",
+        "hide": 0,
+        "includeAll": false,
+        "label": "Datacenter",
+        "multi": false,
+        "name": "datacenter",
+        "options": [],
+        "query": "SELECT DISTINCT datacenter FROM grafana_metric",
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 1,
+        "tagValuesQuery": "",
+        "tags": [],
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      }
+    ]
+  }
+}

public/app/features/dashboard/components/ShareModal/ShareExport.tsx

@@ -1,11 +1,13 @@
 import React, { PureComponent } from 'react';
 import { saveAs } from 'file-saver';
+import { getBackendSrv } from 'app/core/services/backend_srv';
 import { Button, Field, Modal, Switch } from '@grafana/ui';
 import { DashboardModel, PanelModel } from 'app/features/dashboard/state';
 import { DashboardExporter } from 'app/features/dashboard/components/DashExportModal';
 import { appEvents } from 'app/core/core';
 import { ShowModalReactEvent } from 'app/types/events';
 import { ViewJsonModal } from './ViewJsonModal';
+import { config } from '@grafana/runtime';
 
 interface Props {
   dashboard: DashboardModel;
@@ -15,6 +17,7 @@ interface Props {
 
 interface State {
   shareExternally: boolean;
+  trimDefaults: boolean;
 }
 
 export class ShareExport extends PureComponent<Props, State> {
@@ -24,6 +27,7 @@ export class ShareExport extends PureComponent<Props, State> {
     super(props);
     this.state = {
       shareExternally: false,
+      trimDefaults: false,
     };
 
     this.exporter = new DashboardExporter();
@@ -35,29 +39,69 @@ export class ShareExport extends PureComponent<Props, State> {
     });
   };
 
+  onTrimDefaultsChange = () => {
+    this.setState({
+      trimDefaults: !this.state.trimDefaults,
+    });
+  };
+
   onSaveAsFile = () => {
     const { dashboard } = this.props;
     const { shareExternally } = this.state;
+    const { trimDefaults } = this.state;
 
     if (shareExternally) {
       this.exporter.makeExportable(dashboard).then((dashboardJson: any) => {
-        this.openSaveAsDialog(dashboardJson);
+        if (trimDefaults) {
+          getBackendSrv()
+            .post('/api/dashboards/trim', { dashboard: dashboardJson })
+            .then((resp: any) => {
+              this.openSaveAsDialog(resp.dashboard);
+            });
+        } else {
+          this.openSaveAsDialog(dashboardJson);
+        }
       });
     } else {
-      this.openSaveAsDialog(dashboard.getSaveModelClone());
+      if (trimDefaults) {
+        getBackendSrv()
+          .post('/api/dashboards/trim', { dashboard: dashboard.getSaveModelClone() })
+          .then((resp: any) => {
+            this.openSaveAsDialog(resp.dashboard);
+          });
+      } else {
+        this.openSaveAsDialog(dashboard.getSaveModelClone());
+      }
     }
   };
 
   onViewJson = () => {
     const { dashboard } = this.props;
     const { shareExternally } = this.state;
+    const { trimDefaults } = this.state;
 
     if (shareExternally) {
       this.exporter.makeExportable(dashboard).then((dashboardJson: any) => {
-        this.openJsonModal(dashboardJson);
+        if (trimDefaults) {
+          getBackendSrv()
+            .post('/api/dashboards/trim', { dashboard: dashboardJson })
+            .then((resp: any) => {
+              this.openJsonModal(resp.dashboard);
+            });
+        } else {
+          this.openJsonModal(dashboardJson);
+        }
       });
     } else {
-      this.openJsonModal(dashboard.getSaveModelClone());
+      if (trimDefaults) {
+        getBackendSrv()
+          .post('/api/dashboards/trim', { dashboard: dashboard.getSaveModelClone() })
+          .then((resp: any) => {
+            this.openJsonModal(resp.dashboard);
+          });
+      } else {
+        this.openJsonModal(dashboard.getSaveModelClone());
+      }
     }
   };
 
@@ -86,6 +130,7 @@ export class ShareExport extends PureComponent<Props, State> {
   render() {
     const { onDismiss } = this.props;
     const { shareExternally } = this.state;
+    const { trimDefaults } = this.state;
 
     return (
       <>
@@ -93,6 +138,11 @@ export class ShareExport extends PureComponent<Props, State> {
         <Field label="Export for sharing externally">
           <Switch value={shareExternally} onChange={this.onShareExternallyChange} />
         </Field>
+        {config.featureToggles.trimDefaults && (
+          <Field label="Export with trimed dashboard json">
+            <Switch value={trimDefaults} onChange={this.onTrimDefaultsChange} />
+          </Field>
+        )}
         <Modal.ButtonRow>
           <Button variant="secondary" onClick={onDismiss} fill="outline">
             Cancel