Enterprise changes to the Drone pipelines (#33773)

* Enterprise changes to the Drone pipelines
This is basically a no-op in this repository, except for the fact that the grafanabot personal access token will now be fetched from Vault instead of repository secrets
This will pave the way for us to fetch all secrets from Vault

* Update star files from enterprise

* Add missingn newline

.drone.star

@@ -2,8 +2,9 @@ load('scripts/pr.star', 'pr_pipelines')
 load('scripts/main.star', 'main_pipelines')
 load('scripts/release.star', 'release_pipelines', 'test_release_pipelines')
 load('scripts/version.star', 'version_branch_pipelines')
+load('scripts/vault.star', 'secrets')
 
 def main(ctx):
     edition = 'oss'
     return pr_pipelines(edition=edition) + main_pipelines(edition=edition) + release_pipelines() + \
-        test_release_pipelines() + version_branch_pipelines()
+        test_release_pipelines() + version_branch_pipelines() + secrets()

.drone.yml

@@ -1476,6 +1476,9 @@ services:
 - name: memcached
   image: memcached:1.6.9-alpine
 
+image_pull_secrets:
+- dockerconfigjson
+
 trigger:
   ref:
   - refs/tags/v*
@@ -1543,6 +1546,9 @@ steps:
   depends_on:
   - initialize
 
+image_pull_secrets:
+- dockerconfigjson
+
 trigger:
   ref:
   - refs/tags/v*
@@ -2408,6 +2414,9 @@ services:
 - name: memcached
   image: memcached:1.6.9-alpine
 
+image_pull_secrets:
+- dockerconfigjson
+
 trigger:
   event:
   - custom
@@ -2475,6 +2484,9 @@ steps:
   depends_on:
   - initialize
 
+image_pull_secrets:
+- dockerconfigjson
+
 trigger:
   event:
   - custom
@@ -3314,6 +3326,9 @@ services:
 - name: memcached
   image: memcached:1.6.9-alpine
 
+image_pull_secrets:
+- dockerconfigjson
+
 trigger:
   ref:
   - refs/heads/v*
@@ -3377,6 +3392,9 @@ steps:
   depends_on:
   - initialize
 
+image_pull_secrets:
+- dockerconfigjson
+
 trigger:
   ref:
   - refs/heads/v*
@@ -3414,4 +3432,20 @@ depends_on:
 - enterprise-build-release-branch
 - enterprise-windows-release-branch
 
+---
+kind: secret
+name: dockerconfigjson
+
+get:
+  path: secret/data/common/gcr
+  name: .dockerconfigjson
+
+---
+kind: secret
+name: github_token
+
+get:
+  path: infra/data/ci/github/grafanabot
+  name: pat
+
 ...

scripts/lib.star

@@ -1,7 +1,10 @@
+load('scripts/vault.star', 'from_secret', 'github_token', 'pull_secret')
+
 grabpl_version = '2.0.0'
 build_image = 'grafana/build-container:1.4.1'
 publish_image = 'grafana/grafana-ci-deploy:1.3.1'
 grafana_docker_image = 'grafana/drone-grafana-docker:0.3.2'
+deploy_docker_image = 'us.gcr.io/kubernetes-dev/drone/plugins/deploy-image'
 alpine_image = 'alpine:3.13'
 windows_image = 'mcr.microsoft.com/windows:1809'
 dockerize_version = '0.6.1'
@@ -38,6 +41,7 @@ def pipeline(
     }
 
     if edition in ('enterprise', 'enterprise2'):
+        pipeline['image_pull_secrets'] = [pull_secret]
         # We have a custom clone step for enterprise
         pipeline['clone'] = {
             'disable': True,
@@ -67,9 +71,7 @@ def slack_step(channel):
         'name': 'slack',
         'image': 'plugins/slack',
         'settings': {
-            'webhook': {
+            'webhook': from_secret('slack_webhook'),
-                'from_secret': 'slack_webhook',
-            },
             'channel': channel,
             'template': 'Build {{build.number}} failed for commit: <https://github.com/{{repo.owner}}/{{repo.name}}/commit/{{build.commit}}|{{ truncate build.commit 8 }}>: {{build.link}}\nAuthor: {{build.author}}',
         },
@@ -137,9 +139,7 @@ def init_steps(edition, platform, ver_mode, is_downstream=False, install_deps=Tr
                 'name': 'clone',
                 'image': build_image,
                 'environment': {
-                    'GITHUB_TOKEN': {
+                    'GITHUB_TOKEN': from_secret(github_token),
-                        'from_secret': 'github_token',
-                    },
                 },
                 'commands': download_grabpl_cmds + [
                     'git clone "https://$${GITHUB_TOKEN}@github.com/grafana/grafana-enterprise.git"',
@@ -192,9 +192,7 @@ def enterprise_downstream_step(edition):
         'image': 'grafana/drone-downstream',
         'settings': {
             'server': 'https://drone.grafana.net',
-            'token': {
+            'token': from_secret('drone_token'),
-                'from_secret': 'drone_token',
-            },
             'repositories': [
                 'grafana/grafana-enterprise@main',
             ],
@@ -300,9 +298,7 @@ def publish_storybook_step(edition, ver_mode):
             'end-to-end-tests',
         ],
         'environment': {
-            'GCP_KEY': {
+            'GCP_KEY': from_secret('gcp_key'),
-                'from_secret': 'gcp_key',
-            },
         },
         'commands': commands,
     }
@@ -315,9 +311,7 @@ def upload_cdn(edition):
             'package' + enterprise2_sfx(edition),
         ],
         'environment': {
-            'GCP_GRAFANA_UPLOAD_KEY': {
+            'GCP_GRAFANA_UPLOAD_KEY': from_secret('gcp_key'),
-                'from_secret': 'gcp_key',
-            },
         },
         'commands': [
              './bin/grabpl upload-cdn --edition {} --bucket "grafana-static-assets"'.format(edition),
@@ -332,9 +326,7 @@ def build_backend_step(edition, ver_mode, variants=None, is_downstream=False):
     # TODO: Convert number of jobs to percentage
     if ver_mode == 'release':
         env = {
-            'GITHUB_TOKEN': {
+            'GITHUB_TOKEN': from_secret(github_token),
-                'from_secret': 'github_token',
-            },
         }
         cmds = [
             './bin/grabpl build-backend --jobs 8 --edition {} --github-token $${{GITHUB_TOKEN}} --no-pull-enterprise ${{DRONE_TAG}}'.format(
@@ -343,9 +335,7 @@ def build_backend_step(edition, ver_mode, variants=None, is_downstream=False):
         ]
     elif ver_mode == 'test-release':
         env = {
-            'GITHUB_TOKEN': {
+            'GITHUB_TOKEN': from_secret(github_token),
-                'from_secret': 'github_token',
-            },
         }
         cmds = [
             './bin/grabpl build-backend --jobs 8 --edition {} --github-token $${{GITHUB_TOKEN}} --no-pull-enterprise {}'.format(
@@ -424,9 +414,7 @@ def build_frontend_docs_step(edition):
 def build_plugins_step(edition, sign=False):
     if sign:
         env = {
-            'GRAFANA_API_KEY': {
+            'GRAFANA_API_KEY': from_secret('grafana_api_key'),
-                'from_secret': 'grafana_api_key',
-            },
         }
         sign_args = ' --sign --signing-admin'
     else:
@@ -489,9 +477,7 @@ def frontend_metrics_step(edition):
             'initialize',
         ],
         'environment': {
-            'GRAFANA_MISC_STATS_API_KEY': {
+            'GRAFANA_MISC_STATS_API_KEY': from_secret('grafana_misc_stats_api_key'),
-                'from_secret': 'grafana_misc_stats_api_key',
-            },
         },
         'failure': 'ignore',
         'commands': [
@@ -572,21 +558,11 @@ def package_step(edition, ver_mode, variants=None, is_downstream=False):
     if ver_mode in ('main', 'release', 'test-release', 'release-branch'):
         sign_args = ' --sign'
         env = {
-            'GRAFANA_API_KEY': {
+            'GRAFANA_API_KEY': from_secret('grafana_api_key'),
-                'from_secret': 'grafana_api_key',
+            'GITHUB_TOKEN': from_secret(github_token),
-            },
+            'GPG_PRIV_KEY': from_secret('gpg_priv_key'),
-            'GITHUB_TOKEN': {
+            'GPG_PUB_KEY': from_secret('gpg_pub_key'),
-                'from_secret': 'github_token',
+            'GPG_KEY_PASSWORD': from_secret('gpg_key_password'),
-            },
-            'GPG_PRIV_KEY': {
-                'from_secret': 'gpg_priv_key',
-            },
-            'GPG_PUB_KEY': {
-                'from_secret': 'gpg_pub_key',
-            },
-            'GPG_KEY_PASSWORD': {
-                'from_secret': 'gpg_key_password',
-            },
         }
         test_args = ''
     else:
@@ -724,12 +700,8 @@ def build_docker_images_step(edition, ver_mode, archs=None, ubuntu=False, publis
     }
 
     if publish:
-        settings['username'] = {
+        settings['username'] = from_secret('docker_user')
-            'from_secret': 'docker_user',
+        settings['password'] = from_secret('docker_password')
-        }
-        settings['password'] = {
-            'from_secret': 'docker_password',
-        }
     if archs:
         settings['archs'] = ','.join(archs)
     return {
@@ -832,33 +804,31 @@ def release_canary_npm_packages_step(edition):
             'end-to-end-tests',
         ],
         'environment': {
-            'GITHUB_PACKAGE_TOKEN': {
+            'GITHUB_PACKAGE_TOKEN': from_secret('github_package_token'),
-                'from_secret': 'github_package_token',
-            },
         },
         'commands': [
             './scripts/circle-release-canary-packages.sh',
         ],
     }
 
-def deploy_to_kubernetes_step(edition, is_downstream=False):
+def push_to_deployment_tools_step(edition, is_downstream=False):
     if edition != 'enterprise' or not is_downstream:
         return None
 
     return {
-        'name': 'deploy-to-kubernetes',
+        'name': 'push-to-deployment_tools',
-        'image': alpine_image,
+        'image': deploy_docker_image,
         'depends_on': [
             'build-docker-images',
+            # This step should have all the dependencies required for packaging, and should generate
+            # dist/grafana.version
+            'gen-version',
         ],
-        'environment': {
+        'settings': {
-            'CIRCLE_TOKEN': {
+            'github_token': from_secret(github_token),
-                'from_secret': 'deployment_tools_circle_token',
+            'images_file': './deployment_tools_config.json',
+            'docker_tag_file': './dist/grafana.version'
         },
-        },
-        'commands': [
-            './bin/grabpl deploy-to-k8s',
-        ],
     }
 
 def enterprise2_sfx(edition):
@@ -894,9 +864,7 @@ def upload_packages_step(edition, ver_mode, is_downstream=False):
         'image': publish_image,
         'depends_on': dependencies,
         'environment': {
-            'GCP_GRAFANA_UPLOAD_KEY': {
+            'GCP_GRAFANA_UPLOAD_KEY': from_secret('gcp_key'),
-                'from_secret': 'gcp_key',
-            },
         },
         'commands': [cmd,],
     }
@@ -930,21 +898,11 @@ def publish_packages_step(edition, ver_mode, is_downstream=False):
             'initialize',
         ],
         'environment': {
-            'GRAFANA_COM_API_KEY': {
+            'GRAFANA_COM_API_KEY': from_secret('grafana_api_key'),
-                'from_secret': 'grafana_api_key',
+            'GCP_KEY': from_secret('gcp_key'),
-            },
+            'GPG_PRIV_KEY': from_secret('gpg_priv_key'),
-            'GCP_KEY': {
+            'GPG_PUB_KEY': from_secret('gpg_pub_key'),
-                'from_secret': 'gcp_key',
+            'GPG_KEY_PASSWORD': from_secret('gpg_key_password'),
-            },
-            'GPG_PRIV_KEY': {
-                'from_secret': 'gpg_priv_key',
-            },
-            'GPG_PUB_KEY': {
-                'from_secret': 'gpg_pub_key',
-            },
-            'GPG_KEY_PASSWORD': {
-                'from_secret': 'gpg_key_password',
-            },
         },
         'commands': [
             'printenv GCP_KEY | base64 -d > /tmp/gcpkey.json',
@@ -1017,9 +975,7 @@ def get_windows_steps(edition, ver_mode, is_downstream=False):
             'name': 'build-windows-installer',
             'image': wix_image,
             'environment': {
-                'GCP_KEY': {
+                'GCP_KEY': from_secret('gcp_key'),
-                    'from_secret': 'gcp_key',
-                },
             },
             'commands': installer_commands,
             'depends_on': [
@@ -1053,9 +1009,7 @@ def get_windows_steps(edition, ver_mode, is_downstream=False):
             'name': 'clone',
             'image': wix_image,
             'environment': {
-                'GITHUB_TOKEN': {
+                'GITHUB_TOKEN': from_secret(github_token),
-                    'from_secret': 'github_token',
-                },
             },
             'commands': download_grabpl_cmds + clone_cmds,
         })

scripts/main.star

@@ -29,7 +29,7 @@ load(
     'publish_storybook_step',
     'release_canary_npm_packages_step',
     'upload_packages_step',
-    'deploy_to_kubernetes_step',
+    'push_to_deployment_tools_step',
     'publish_packages_step',
     'notify_pipeline',
     'integration_test_services',
@@ -87,7 +87,7 @@ def get_steps(edition, is_downstream=False):
     steps.extend([
         release_canary_npm_packages_step(edition),
         upload_packages_step(edition=edition, ver_mode=ver_mode, is_downstream=is_downstream),
-        deploy_to_kubernetes_step(edition=edition, is_downstream=is_downstream),
+        push_to_deployment_tools_step(edition=edition, is_downstream=is_downstream),
         upload_cdn(edition=edition)
     ])
 

scripts/vault.star

@@ -0,0 +1,23 @@
+pull_secret = 'dockerconfigjson'
+github_token = 'github_token'
+
+def from_secret(secret):
+    return {
+        'from_secret': secret
+    }
+
+def vault_secret(name, path, key):
+    return {
+        'kind': 'secret',
+        'name': name,
+        'get': {
+            'path': path,
+            'name': key,
+        }
+    }
+
+def secrets():
+    return [
+        vault_secret(pull_secret, 'secret/data/common/gcr', '.dockerconfigjson'),
+        vault_secret(github_token, 'infra/data/ci/github/grafanabot', 'pat')
+    ]