E2E: Run playwright cloud plugins tests as part of github actions (#109055)

* add github workflow scaffolding

* update comments

* Add image and resource commands

* Add secrets paths

* Block workflow run for forks

* ignore via package.json, update CODEOWNERS

* fix workflow path

* remove old azure monitor test

* pull docker image first

* add permissions for docker pull step

* checkout first

* keep creds file

* try all in one job

* with creds...

* add cloud: 'azure'

* pass CLOUD to docker

* add -playwright

* actually use the env vars

* don't need to pass CLOUD env var

* remove commented out code and tidy up

* kick CI

* Update container names and set PLAYWRIGHT_CI

* Update path

* fix zizmor violation

* use bigger runner, add double quoting

* add separate command and increase timeout

* remove timeout

* parameterise the e2e command in CI

* move cloud-plugins-e2e-tests into normal e2e test workflow

* fix detect changes

* pass creds into dagger

* try remove quotes

* add a debug log

* exec playwright command after mounting file

* reassign e2eContainer, add change to check the tests fail correctly

* fix test

---------

Co-authored-by: Andreas Christou <andreas.christou@grafana.com>

.github/CODEOWNERS

@@ -412,8 +412,8 @@
 /public/locales/i18next-parser-enterprise.config.cjs @grafana/grafana-frontend-platform
 /public/app/core/internationalization/ @grafana/grafana-frontend-platform
 /e2e/ @grafana/grafana-frontend-platform
-/e2e/cloud-plugins-suite/ @grafana/partner-datasources
 /e2e-playwright/ @grafana/grafana-frontend-platform
+/e2e-playwright/cloud-plugins-suite/ @grafana/partner-datasources
 /e2e-playwright/dashboard-new-layouts @grafana/dashboards-squad
 /e2e-playwright/dashboards-suite/dashboard-browse-nested.spec.ts @grafana/grafana-search-navigate-organise
 /e2e-playwright/dashboards-suite/dashboard-browse.spec.ts @grafana/grafana-search-navigate-organise

.github/actions/change-detection/action.yml

@@ -19,6 +19,9 @@ outputs:
     value: ${{ steps.changed-files.outputs.e2e_any_changed == 'true' ||
       steps.changed-files.outputs.backend_any_changed == 'true' ||
       steps.changed-files.outputs.frontend_any_changed == 'true' || 'true' }}
+  e2e-cloud-plugins:
+    description: Whether the cloud plugins code or tests have changed in any way
+    value: ${{ steps.changed-files.outputs.e2e_cloud_plugins_any_changed || 'true' }}
   dev-tooling:
     description: Whether the dev tooling or self have changed in any way
     value: ${{ steps.changed-files.outputs.dev_tooling_any_changed || 'true' }}
@@ -102,6 +105,11 @@ runs:
             - 'conf/**'
             - 'cypress.config.js'
             - '${{ inputs.self }}'
+          e2e_cloud_plugins:
+            - 'pkg/tsdb/azuremonitor/**'
+            - 'public/app/plugins/datasource/azuremonitor/**'
+            - 'e2e-playwright/cloud-plugins-suite/azure-monitor.spec.ts'
+            - '${{ inputs.self }}'
           dev_tooling:
             - '.github/actions/setup-enterprise/**'
             - '.github/actions/checkout/**'
@@ -139,6 +147,8 @@ runs:
         echo " --> ${{ steps.changed-files.outputs.e2e_all_changed_files }}"
         echo " --> ${{ steps.changed-files.outputs.backend_all_changed_files }}"
         echo " --> ${{ steps.changed-files.outputs.frontend_all_changed_files }}"
+        echo "E2E cloud plugins: ${{ steps.changed-files.outputs.e2e_cloud_plugins_any_changed || 'true' }}"
+        echo " --> ${{ steps.changed-files.outputs.e2e_cloud_plugins_all_changed_files }}"
         echo "Dev Tooling: ${{ steps.changed-files.outputs.dev_tooling_any_changed || 'true' }}"
         echo " --> ${{ steps.changed-files.outputs.dev_tooling_all_changed_files }}"
         echo "Docs: ${{ steps.changed-files.outputs.docs_any_changed || 'true' }}"

.github/workflows/pr-e2e-tests.yml

@@ -26,6 +26,7 @@ jobs:
       contents: read
     outputs:
       changed: ${{ steps.detect-changes.outputs.e2e }}
+      cloud_plugins_changed: ${{ steps.detect-changes.outputs.e2e-cloud-plugins }}
     steps:
       - uses: actions/checkout@v4
         with:
@@ -318,9 +319,76 @@ jobs:
           path: ./blob-report
           retention-days: 1
 
+  run-azure-monitor-e2e:
+    if: needs.detect-changes.outputs.cloud_plugins_changed == 'true' && github.event.pull_request.head.repo.fork == false
+    runs-on: ubuntu-x64-large
+    needs:
+      - build-grafana
+      - detect-changes
+    permissions:
+      contents: read
+      id-token: write
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+
+      - uses: grafana/shared-workflows/actions/login-to-gar@login-to-gar-v0.4.0
+        id: login-to-gar
+        with:
+          registry: "us-docker.pkg.dev"
+          environment: "dev"
+
+      - id: pull-docker-image
+        run: |
+          docker pull us-docker.pkg.dev/grafanalabs-dev/docker-oss-plugin-partnerships-dev/e2e-playwright:latest
+
+      - id: vault-secrets
+        uses: grafana/shared-workflows/actions/get-vault-secrets@main
+        with:
+          repo_secrets: |
+            AZURE_SP_APP_ID=cpp-azure-resourcemanager-credentials:application_id
+            AZURE_SP_PASSWORD=cpp-azure-resourcemanager-credentials:application_secret
+            AZURE_TENANT=cpp-azure-resourcemanager-credentials:tenant_id
+
+      - id: deploy-resources
+        env:
+          AZURE_SP_APP_ID: ${{ env.AZURE_SP_APP_ID}}
+          AZURE_SP_PASSWORD: ${{ env.AZURE_SP_PASSWORD}}
+          AZURE_TENANT: ${{ env.AZURE_TENANT }}
+          NAME: ${{ github.ref_name }}
+        run: |
+          docker container run --name cpp-e2e-deploy -e AZURE_SP_APP_ID -e AZURE_SP_PASSWORD -e AZURE_TENANT -e PLAYWRIGHT_CI=true us-docker.pkg.dev/grafanalabs-dev/docker-oss-plugin-partnerships-dev/e2e-playwright:latest ./cpp-e2e/scripts/ci-run-playwright.sh azure "${NAME}" deploy
+
+      - id: extract-creds
+        # see https://github.com/grafana/oss-plugin-partnerships/blob/a77040d0456003cd258668b61d542dc7c75db5b5/e2e/scripts/deploy.sh#L25 for path
+        run: |
+          docker cp cpp-e2e-deploy:/outputs.json /tmp/outputs.json
+
+      - uses: actions/download-artifact@v4
+        with:
+          name: grafana-tar-gz
+
+      - name: Run E2E tests
+        uses: dagger/dagger-for-github@e47aba410ef9bb9ed81a4d2a97df31061e5e842e
+        with:
+          verb: run
+          args: go run ./pkg/build/e2e-playwright --package=grafana.tar.gz --playwright-command="yarn e2e:playwright:cloud-plugins" --cloud-plugin-creds=/tmp/outputs.json
+
+      - name: Destroy resources
+        if: always() && steps.deploy-resources.outcome == 'success'
+        env:
+          AZURE_SP_APP_ID: ${{ env.AZURE_SP_APP_ID }}
+          AZURE_SP_PASSWORD: ${{ env.AZURE_SP_PASSWORD }}
+          AZURE_TENANT: ${{ env.AZURE_TENANT }}
+          NAME: ${{ github.ref_name }}
+        run: |
+          docker container run --name cpp-e2e-destroy -e AZURE_SP_APP_ID -e AZURE_SP_PASSWORD -e AZURE_TENANT us-docker.pkg.dev/grafanalabs-dev/docker-oss-plugin-partnerships-dev/e2e-playwright:latest ./cpp-e2e/scripts/ci-run-playwright.sh azure "${NAME}" destroy
+
   required-playwright-tests:
     needs:
       - run-playwright-tests
+      - run-azure-monitor-e2e
       - run-storybook-test
       - build-grafana
     if: ${{ !cancelled() }}

e2e-playwright/cloud-plugins-suite/azure-monitor.spec.ts

@@ -4,12 +4,12 @@ import { Page } from 'playwright-core';
 import { v4 as uuidv4 } from 'uuid';
 
 import {
-  test,
-  expect,
   CreateDataSourcePageArgs,
+  DashboardPage,
   DataSourceConfigPage,
   E2ESelectorGroups,
-  DashboardPage,
+  expect,
+  test,
 } from '@grafana/plugin-e2e';
 
 import { AzureQueryType } from '../../public/app/plugins/datasource/azuremonitor/dataquery.gen';
@@ -71,8 +71,7 @@ async function provisionAzureMonitorDatasources(
   await configPage.saveAndTest();
 }
 
-// TODO unskip when we've figured out how to populate the credentials in CI
-test.describe.skip(
+test.describe(
   'Azure Monitor datasource',
   {
     tag: ['@cloud-plugins'],
@@ -84,7 +83,7 @@ test.describe.skip(
       // Check if we're running in CI
       const CI = process.env.CI;
       if (CI) {
-        const outputs = JSON.parse(readFileSync('outputs.json', 'utf8'));
+        const outputs = JSON.parse(readFileSync('/tmp/outputs.json', 'utf8'));
         datasourceConfig = {
           jsonData: {
             cloudName: 'Azure',
@@ -134,6 +133,7 @@ test.describe.skip(
       await expect(page.getByText(rootSubscription)).toBeVisible({ timeout: 30000 });
       const resourceSearchInput = page.getByTestId(azMonSelectors.components.queryEditor.resourcePicker.search.input);
       await resourceSearchInput.fill(storageAcctName);
+      await resourceSearchInput.press('Enter');
       await expect(page.getByText(storageAcctName)).toBeVisible({ timeout: 30000 });
       await page.getByText(storageAcctName).click();
       const applyButton = page.getByTestId(azMonSelectors.components.queryEditor.resourcePicker.apply.button);
@@ -164,6 +164,7 @@ test.describe.skip(
       await resourcePickerButton.click();
       await expect(page.getByText(rootSubscription)).toBeVisible({ timeout: 30000 });
       await resourceSearchInput.fill(logAnalyticsName);
+      await resourceSearchInput.press('Enter');
       await expect(page.getByText(logAnalyticsName)).toBeVisible({ timeout: 30000 });
       await page.getByText(logAnalyticsName).click();
       await applyButton.click();
@@ -220,6 +221,7 @@ test.describe.skip(
       await resourcePickerButton.click();
       await expect(page.getByText(rootSubscription)).toBeVisible({ timeout: 30000 });
       await resourceSearchInput.fill(applicationInsightsName);
+      await resourceSearchInput.press('Enter');
       await expect(page.getByText(applicationInsightsName)).toBeVisible({ timeout: 30000 });
       await page.getByText(applicationInsightsName).click();
       await applyButton.click();

e2e/cloud-plugins-suite/azure-monitor.spec.ts

@@ -1,360 +0,0 @@
-import { Interception } from 'cypress/types/net-stubbing';
-import { load } from 'js-yaml';
-import { v4 as uuidv4 } from 'uuid';
-
-import { selectors as rawSelectors } from '@grafana/e2e-selectors';
-
-import { selectors } from '../../public/app/plugins/datasource/azuremonitor/e2e/selectors';
-import { AzureQueryType } from '../../public/app/plugins/datasource/azuremonitor/types/query';
-import {
-  AzureMonitorDataSourceJsonData,
-  AzureMonitorDataSourceSecureJsonData,
-} from '../../public/app/plugins/datasource/azuremonitor/types/types';
-import { e2e } from '../utils';
-
-const provisioningPath = `provisioning/datasources/azmonitor-ds.yaml`;
-const e2eSelectors = e2e.getSelectors(selectors.components);
-
-type AzureMonitorConfig = {
-  secureJsonData: AzureMonitorDataSourceSecureJsonData;
-  jsonData: AzureMonitorDataSourceJsonData;
-};
-
-type AzureMonitorProvision = { datasources: AzureMonitorConfig[] };
-
-const dataSourceName = `Azure Monitor E2E Tests - ${uuidv4()}`;
-
-const maxRetryCount = 3;
-
-Cypress.Commands.add('checkHealthRetryable', function (fn: Function, retryCount: number) {
-  cy.then(() => {
-    const result = fn(++retryCount);
-    result.then((res: Interception) => {
-      if (retryCount < maxRetryCount && res.response.statusCode !== 200) {
-        cy.wait(20000);
-        cy.checkHealthRetryable(fn, retryCount);
-      }
-    });
-  });
-});
-
-function provisionAzureMonitorDatasources(datasources: AzureMonitorProvision[]) {
-  const datasource = datasources[0].datasources[0];
-
-  cy.intercept(/subscriptions/).as('subscriptions');
-
-  e2e.flows.addDataSource({
-    type: 'Azure Monitor',
-    name: dataSourceName,
-    form: () => {
-      e2eSelectors.configEditor.azureCloud.input().find('input').type('Azure').type('{enter}');
-      // We set the log value to false here to ensure that secrets aren't printed to logs
-      e2eSelectors.configEditor.tenantID.input().find('input').type(datasource.jsonData.tenantId, { log: false });
-      e2eSelectors.configEditor.clientID.input().find('input').type(datasource.jsonData.clientId, { log: false });
-      e2eSelectors.configEditor.clientSecret
-        .input()
-        .find('input')
-        .type(datasource.secureJsonData.clientSecret, { log: false });
-      e2eSelectors.configEditor.loadSubscriptions.button().click().wait('@subscriptions').wait(500);
-      e2eSelectors.configEditor.defaultSubscription.input().find('input').type('datasources{enter}');
-
-      // We can do this because awaitHealth is set to true so @health is defined
-      cy.checkHealthRetryable(() => {
-        return e2e.pages.DataSource.saveAndTest().click().wait('@health');
-      }, 0);
-    },
-    expectedAlertMessage: 'Successfully connected to all Azure Monitor endpoints',
-    // Reduce the timeout from 30s to error faster when an invalid alert message is presented
-    timeout: 10000,
-    awaitHealth: true,
-  });
-}
-
-// Helper function to add template variables
-const addAzureMonitorVariable = (
-  name: string,
-  type: AzureQueryType,
-  isFirst: boolean,
-  options?: { subscription?: string; resourceGroup?: string; namespace?: string; resource?: string; region?: string }
-) => {
-  e2e.components.NavToolbar.editDashboard.editButton().should('be.visible').click();
-  e2e.components.NavToolbar.editDashboard.settingsButton().should('be.visible').click();
-  e2e.components.Tab.title('Variables').click();
-  if (isFirst) {
-    e2e.pages.Dashboard.Settings.Variables.List.addVariableCTAV2().click();
-  } else {
-    cy.get(`[data-testid="${rawSelectors.pages.Dashboard.Settings.Variables.List.newButton}"]`).click();
-  }
-  e2e.pages.Dashboard.Settings.Variables.Edit.General.generalNameInputV2().clear().type(name);
-  e2e.components.DataSourcePicker.inputV2().type(`${dataSourceName}{enter}`);
-  e2eSelectors.variableEditor.queryType
-    .input()
-    .find('input')
-    .type(`${type.replace('Azure', '').trim()}{enter}`);
-  switch (type) {
-    case AzureQueryType.ResourceGroupsQuery:
-      e2eSelectors.variableEditor.subscription.input().find('input').type(`${options?.subscription}{enter}`);
-      break;
-    case AzureQueryType.LocationsQuery:
-      e2eSelectors.variableEditor.subscription.input().find('input').type(`${options?.subscription}{enter}`);
-      break;
-    case AzureQueryType.NamespacesQuery:
-      e2eSelectors.variableEditor.subscription.input().find('input').type(`${options?.subscription}{enter}`);
-      e2eSelectors.variableEditor.resourceGroup.input().find('input').type(`${options?.resourceGroup}{enter}`);
-      break;
-    case AzureQueryType.ResourceNamesQuery:
-      e2eSelectors.variableEditor.subscription.input().find('input').type(`${options?.subscription}{enter}`);
-      e2eSelectors.variableEditor.resourceGroup.input().find('input').type(`${options?.resourceGroup}{enter}`);
-      e2eSelectors.variableEditor.namespace.input().find('input').type(`${options?.namespace}{enter}`);
-      e2eSelectors.variableEditor.region.input().find('input').type(`${options?.region}{enter}`);
-      break;
-    case AzureQueryType.MetricNamesQuery:
-      e2eSelectors.variableEditor.subscription.input().find('input').type(`${options?.subscription}{enter}`);
-      e2eSelectors.variableEditor.resourceGroup.input().find('input').type(`${options?.resourceGroup}{enter}`);
-      e2eSelectors.variableEditor.namespace.input().find('input').type(`${options?.namespace}{enter}`);
-      e2eSelectors.variableEditor.resource.input().find('input').type(`${options?.resource}{enter}`);
-      break;
-  }
-  e2e.pages.Dashboard.Settings.Variables.Edit.General.submitButton().click();
-  e2e.components.NavToolbar.editDashboard.backToDashboardButton().click();
-  e2e.components.NavToolbar.editDashboard.exitButton().click();
-};
-
-const storageAcctName = 'azmonteststorage';
-const logAnalyticsName = 'az-mon-test-logs';
-const applicationInsightsName = 'az-mon-test-ai-a';
-
-describe('Azure monitor datasource', () => {
-  before(() => {
-    e2e.flows.login(Cypress.env('USERNAME'), Cypress.env('PASSWORD'));
-
-    // Add datasource
-    // This variable will be set in CI
-    const CI = Cypress.env('CI');
-    if (CI) {
-      cy.readFile('outputs.json').then((outputs) => {
-        provisionAzureMonitorDatasources([
-          {
-            datasources: [
-              {
-                jsonData: {
-                  cloudName: 'Azure',
-                  tenantId: outputs.tenantId,
-                  clientId: outputs.clientId,
-                },
-                secureJsonData: { clientSecret: outputs.clientSecret },
-              },
-            ],
-          },
-        ]);
-      });
-    } else {
-      cy.readFile(provisioningPath).then((azMonitorProvision: string) => {
-        // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
-        const yaml = load(azMonitorProvision) as AzureMonitorProvision;
-        provisionAzureMonitorDatasources([yaml]);
-      });
-    }
-    e2e.setScenarioContext({ addedDataSources: [] });
-  });
-
-  beforeEach(() => {
-    e2e.flows.login(Cypress.env('USERNAME'), Cypress.env('PASSWORD'));
-  });
-
-  after(() => {
-    e2e.flows.login(Cypress.env('USERNAME'), Cypress.env('PASSWORD'));
-    e2e.flows.revertAllChanges();
-  });
-
-  it('create dashboard, add panel for metrics, log analytics, ARG, and traces queries', () => {
-    e2e.flows.addDashboard({
-      timeRange: {
-        from: 'now-6h',
-        to: 'now',
-        zone: 'Coordinated Universal Time',
-      },
-    });
-    e2e.flows.addPanel({
-      dataSourceName,
-      visitDashboardAtStart: false,
-      queriesForm: () => {
-        e2eSelectors.queryEditor.resourcePicker.select.button().click();
-        e2eSelectors.queryEditor.resourcePicker.search
-          .input()
-          .wait(100)
-          .type(storageAcctName)
-          .wait(500)
-          .type('{enter}');
-        cy.contains(storageAcctName).click();
-        e2eSelectors.queryEditor.resourcePicker.apply.button().click();
-        cy.contains('microsoft.storage/storageaccounts');
-        e2eSelectors.queryEditor.metricsQueryEditor.metricName.input().find('input').type('Used capacity{enter}');
-      },
-      timeout: 10000,
-    });
-    e2e.components.NavToolbar.editDashboard.backToDashboardButton().click();
-    e2e.components.NavToolbar.editDashboard.exitButton().click();
-    e2e.flows.addPanel({
-      dataSourceName,
-      visitDashboardAtStart: false,
-      queriesForm: () => {
-        e2eSelectors.queryEditor.header.select().find('input').type('Logs{enter}');
-        e2eSelectors.queryEditor.resourcePicker.select.button().click();
-        e2eSelectors.queryEditor.resourcePicker.search
-          .input()
-          .wait(100)
-          .type(logAnalyticsName)
-          .wait(500)
-          .type('{enter}');
-        cy.contains(logAnalyticsName).click();
-        e2eSelectors.queryEditor.resourcePicker.apply.button().click();
-        e2e.components.CodeEditor.container().type('AzureDiagnostics');
-        e2eSelectors.queryEditor.logsQueryEditor.formatSelection.input().type('Time series{enter}');
-      },
-      timeout: 10000,
-    });
-    e2e.components.NavToolbar.editDashboard.backToDashboardButton().click();
-    e2e.components.NavToolbar.editDashboard.exitButton().click();
-    e2e.flows.addPanel({
-      dataSourceName,
-      visitDashboardAtStart: false,
-      queriesForm: () => {
-        e2eSelectors.queryEditor.header.select().find('input').type('Azure Resource Graph{enter}');
-        cy.wait(2000); // Need to wait for code editor to completely load
-        e2eSelectors.queryEditor.argsQueryEditor.subscriptions.input().find('[aria-label="Clear value"]').click();
-        e2eSelectors.queryEditor.argsQueryEditor.subscriptions.input().find('input').type('datasources{enter}');
-        e2e.components.CodeEditor.container().type(
-          "Resources | where resourceGroup == 'cloud-plugins-e2e-test-azmon' | project name, resourceGroup"
-        );
-        e2e.components.PanelEditor.toggleTableView().click({ force: true });
-      },
-      timeout: 10000,
-    });
-    e2e.components.NavToolbar.editDashboard.backToDashboardButton().click();
-    e2e.components.NavToolbar.editDashboard.exitButton().click();
-    e2e.flows.addPanel({
-      dataSourceName,
-      visitDashboardAtStart: false,
-      queriesForm: () => {
-        e2eSelectors.queryEditor.header.select().find('input').type('Traces{enter}');
-        e2eSelectors.queryEditor.resourcePicker.select.button().click();
-        e2eSelectors.queryEditor.resourcePicker.search
-          .input()
-          .wait(100)
-          .type(applicationInsightsName)
-          .wait(500)
-          .type('{enter}');
-        cy.contains(applicationInsightsName).click();
-        e2eSelectors.queryEditor.resourcePicker.apply.button().click();
-        cy.wait(10000);
-        e2eSelectors.queryEditor.logsQueryEditor.formatSelection.input().type('Trace{enter}');
-      },
-      timeout: 10000,
-    });
-  });
-
-  it('creates a dashboard that includes a template variable', () => {
-    e2e.flows.addDashboard({
-      timeRange: {
-        from: 'now-6h',
-        to: 'now',
-        zone: 'Coordinated Universal Time',
-      },
-    });
-    addAzureMonitorVariable('subscription', AzureQueryType.SubscriptionsQuery, true);
-    addAzureMonitorVariable('resourceGroups', AzureQueryType.ResourceGroupsQuery, false, {
-      subscription: '$subscription',
-    });
-    addAzureMonitorVariable('namespaces', AzureQueryType.NamespacesQuery, false, {
-      subscription: '$subscription',
-      resourceGroup: '$resourceGroups',
-    });
-    addAzureMonitorVariable('region', AzureQueryType.LocationsQuery, false, {
-      subscription: '$subscription',
-    });
-    addAzureMonitorVariable('resource', AzureQueryType.ResourceNamesQuery, false, {
-      subscription: '$subscription',
-      resourceGroup: '$resourceGroups',
-      namespace: '$namespace',
-      region: '$region',
-    });
-    e2e.pages.Dashboard.SubMenu.submenuItemLabels('subscription')
-      .parent()
-      .within(() => {
-        cy.get('input').click();
-      });
-    e2e.components.Select.option().contains('grafanalabs-datasources-dev').click();
-    e2e.pages.Dashboard.SubMenu.submenuItemLabels('resourceGroups')
-      .parent()
-      .within(() => {
-        cy.get('input').type('cloud-plugins-e2e-test-azmon{downArrow}{enter}');
-      });
-    e2e.pages.Dashboard.SubMenu.submenuItemLabels('namespaces')
-      .parent()
-      .within(() => {
-        cy.get('input').type('microsoft.storage/storageaccounts{downArrow}{enter}');
-      });
-    e2e.pages.Dashboard.SubMenu.submenuItemLabels('region')
-      .parent()
-      .within(() => {
-        cy.get('input').type('uk south{downArrow}{enter}');
-      });
-    e2e.pages.Dashboard.SubMenu.submenuItemLabels('resource')
-      .parent()
-      .within(() => {
-        cy.get('input').type(`${storageAcctName}{downArrow}{enter}`);
-      });
-    e2e.flows.addPanel({
-      dataSourceName,
-      visitDashboardAtStart: false,
-      queriesForm: () => {
-        e2eSelectors.queryEditor.resourcePicker.select.button().click();
-        e2eSelectors.queryEditor.resourcePicker.advanced.collapse().click();
-        e2eSelectors.queryEditor.resourcePicker.advanced.subscription.input().find('input').type('$subscription');
-        e2eSelectors.queryEditor.resourcePicker.advanced.resourceGroup.input().find('input').type('$resourceGroups');
-        e2eSelectors.queryEditor.resourcePicker.advanced.namespace.input().find('input').type('$namespaces');
-        e2eSelectors.queryEditor.resourcePicker.advanced.region.input().find('input').type('$region');
-        e2eSelectors.queryEditor.resourcePicker.advanced.resource.input().find('input').type('$resource');
-        e2eSelectors.queryEditor.resourcePicker.apply.button().click();
-        e2eSelectors.queryEditor.metricsQueryEditor.metricName.input().find('input').type('Transactions{enter}');
-      },
-      timeout: 10000,
-    });
-  });
-
-  it.skip('creates a dashboard that includes an annotation', () => {
-    e2e.flows.addDashboard({
-      timeRange: {
-        from: '2022-10-03 00:00:00',
-        to: '2022-10-03 23:59:59',
-        zone: 'Coordinated Universal Time',
-      },
-    });
-    e2e.components.PageToolbar.item('Dashboard settings').click();
-    e2e.components.Tab.title('Annotations').click();
-    e2e.pages.Dashboard.Settings.Annotations.List.addAnnotationCTAV2().click();
-    e2e.pages.Dashboard.Settings.Annotations.Settings.name().type('TestAnnotation');
-    e2e.components.DataSourcePicker.inputV2().click().type(`${dataSourceName}{enter}`);
-    e2eSelectors.queryEditor.resourcePicker.select.button().click();
-    e2eSelectors.queryEditor.resourcePicker.search.input().type(storageAcctName);
-    cy.contains(storageAcctName).click();
-    e2eSelectors.queryEditor.resourcePicker.apply.button().click();
-    cy.contains('microsoft.storage/storageaccounts');
-    e2eSelectors.queryEditor.metricsQueryEditor.metricName.input().find('input').type('Transactions{enter}');
-    cy.get('table').contains('text').parent().find('input').click().type('Transactions (number){enter}');
-    e2e.components.PageToolbar.item('Go Back').click();
-    e2e.flows.addPanel({
-      dataSourceName,
-      visitDashboardAtStart: false,
-      queriesForm: () => {
-        e2eSelectors.queryEditor.resourcePicker.select.button().click();
-        e2eSelectors.queryEditor.resourcePicker.search.input().type(storageAcctName);
-        cy.contains(storageAcctName).click();
-        e2eSelectors.queryEditor.resourcePicker.apply.button().click();
-        cy.contains('microsoft.storage/storageaccounts');
-        e2eSelectors.queryEditor.metricsQueryEditor.metricName.input().find('input').type('Used capacity{enter}');
-      },
-    });
-  });
-});

package.json

@@ -21,7 +21,8 @@
     "e2e:enterprise": "./e2e/start-and-run-suite enterprise",
     "e2e:enterprise:dev": "./e2e/start-and-run-suite enterprise dev",
     "e2e:enterprise:debug": "./e2e/start-and-run-suite enterprise debug",
-    "e2e:playwright": "yarn playwright test",
+    "e2e:playwright": "yarn playwright test --grep-invert @cloud-plugins",
+    "e2e:playwright:cloud-plugins": "yarn playwright test --grep @cloud-plugins",
     "e2e:playwright:storybook": "yarn playwright test -c playwright.storybook.config.ts",
     "e2e:acceptance": "yarn playwright test --grep @acceptance",
     "e2e:storybook": "PORT=9001 ./e2e/run-suite storybook true",

pkg/build/e2e-playwright/e2e.go

@@ -23,6 +23,8 @@ type RunTestOpts struct {
 	HTMLReportExportDir  string
 	BlobReportExportDir  string
 	TestResultsExportDir string
+	PlaywrightCommand    string
+	CloudPluginCreds     *dagger.File
 }
 
 func RunTest(
@@ -47,10 +49,16 @@ func RunTest(
 		WithEnvVariable("bustcache", "1").
 		WithEnvVariable("PLAYWRIGHT_HTML_OPEN", "never").
 		WithEnvVariable("PLAYWRIGHT_HTML_OUTPUT_DIR", htmlResultsDir).
-		WithEnvVariable("PLAYWRIGHT_BLOB_OUTPUT_DIR", blobResultsDir).
-		WithExec(playwrightCommand, dagger.ContainerWithExecOpts{
-			Expect: dagger.ReturnTypeAny,
-		})
+		WithEnvVariable("PLAYWRIGHT_BLOB_OUTPUT_DIR", blobResultsDir)
+
+	if opts.CloudPluginCreds != nil {
+		fmt.Println("DEBUG: CloudPluginCreds file is provided, mounting to /tmp/outputs.json")
+		e2eContainer = e2eContainer.WithMountedFile("/tmp/outputs.json", opts.CloudPluginCreds)
+	}
+
+	e2eContainer = e2eContainer.WithExec(playwrightCommand, dagger.ContainerWithExecOpts{
+		Expect: dagger.ReturnTypeAny,
+	})
 
 	if opts.TestResultsExportDir != "" {
 		_, err := e2eContainer.Directory(testResultsDir).Export(ctx, opts.TestResultsExportDir)
@@ -89,14 +97,14 @@ func buildPlaywrightCommand(opts RunTestOpts) []string {
 		playwrightReporters = append(playwrightReporters, "blob")
 	}
 
-	playwrightCommand := []string{
-		"yarn",
-		"e2e:playwright",
+	playwrightExec := strings.Split(opts.PlaywrightCommand, " ")
+
+	playwrightCommand := append(playwrightExec,
 		"--reporter",
 		strings.Join(playwrightReporters, ","),
 		"--output",
 		testResultsDir,
-	}
+	)
 
 	if opts.Shard != "" {
 		playwrightCommand = append(playwrightCommand, "--shard", opts.Shard)

pkg/build/e2e-playwright/main.go

@@ -71,6 +71,17 @@ func NewApp() *cli.Command {
 				Usage:     "Enables the blob reporter, exported to this directory. Useful with --shard (optional)",
 				Validator: mustBeDir("blob-dir", true, true),
 			},
+			&cli.StringFlag{
+				Name:  "playwright-command",
+				Usage: "The playwright command to run.",
+				Value: "yarn e2e:playwright",
+			},
+			&cli.StringFlag{
+				Name:      "cloud-plugin-creds",
+				Usage:     "Path to the cloud plugin credentials file (only required for running @cloud-plugins e2e tests)",
+				Validator: mustBeFile("cloud-plugin-creds", true),
+				TakesFile: true,
+			},
 		},
 		Action: run,
 	}
@@ -80,10 +91,12 @@ func run(ctx context.Context, cmd *cli.Command) error {
 	grafanaDir := cmd.String("grafana-dir")
 	targzPath := cmd.String("package")
 	licensePath := cmd.String("license")
+	cloudPluginCredsPath := cmd.String("cloud-plugin-creds")
 	pwShard := cmd.String("shard")
 	resultsDir := cmd.String("results-dir")
 	htmlDir := cmd.String("html-dir")
 	blobDir := cmd.String("blob-dir")
+	playwrightCommand := cmd.String("playwright-command")
 	// pa11yConfigPath := cmd.String("config")
 	// pa11yResultsPath := cmd.String("results")
 	// noThresholdFail := cmd.Bool("no-threshold-fail")
@@ -156,6 +169,11 @@ func run(ctx context.Context, cmd *cli.Command) error {
 		license = d.Host().File(licensePath)
 	}
 
+	var cloudPluginCreds *dagger.File
+	if cloudPluginCredsPath != "" {
+		cloudPluginCreds = d.Host().File(cloudPluginCredsPath)
+	}
+
 	svc, err := GrafanaService(ctx, d, GrafanaServiceOpts{
 		HostSrc:           grafanaHostSrc,
 		FrontendContainer: frontendContainer,
@@ -174,6 +192,8 @@ func run(ctx context.Context, cmd *cli.Command) error {
 		TestResultsExportDir: resultsDir,
 		HTMLReportExportDir:  htmlDir,
 		BlobReportExportDir:  blobDir,
+		PlaywrightCommand:    playwrightCommand,
+		CloudPluginCreds:     cloudPluginCreds,
 	}
 
 	c, runErr := RunTest(ctx, d, runOpts)