Package release before publishing (#42218)

* Package separately to publish

* Fix interpolation

* Windows format envvars

* More descriptive msg

* Won't publish from here

* Resolve docker issues in PR build

* Rename package docker step

* Correct npm release JSON structure

Makefile

@@ -158,7 +158,7 @@ gen-ts:
 # you modify starlark files.
 drone: $(DRONE)
 	$(DRONE) starlark --format
-	$(DRONE) lint .drone.yml
+	$(DRONE) lint .drone.yml --trusted
 	$(DRONE) --server https://drone.grafana.net sign --save grafana/grafana
 
 help: ## Display this help.

scripts/build/build-npm-packages.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+# lerna bootstrap might have created yarn.lock
+git checkout .
+
+echo 'Building packages'
+yarn packages:build

scripts/build/release-npm-packages.sh

@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 
 PACKAGES=("@grafana/ui" "@grafana/data" "@grafana/toolkit" "@grafana/runtime" "@grafana/e2e" "@grafana/e2e-selectors" "@grafana/schema")
+
 GRAFANA_TAG=${1:-}
 RELEASE_CHANNEL="latest"
 
@@ -39,9 +40,6 @@ fi
 # Publish to NPM registry
 echo "//registry.npmjs.org/:_authToken=${NPM_TOKEN}" >> ~/.npmrc
 
-echo $'\nBuilding packages'
-yarn packages:build
-
 echo $'\nPublishing packages to NPM registry'
 yarn packages:${SCRIPT}
 

scripts/build/retrieve-npm-packages.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+set -e
+
+GRAFANA_TAG=${1:-}
+
+if echo "$GRAFANA_TAG" | grep -q "^v"; then
+	_grafana_version=$(echo "${GRAFANA_TAG}" | cut -d "v" -f 2)
+else
+  echo "Provided tag is not a version tag, skipping packages release..."
+	exit
+fi
+
+echo "Retrieving prerelease NPM artifacts"
+ZIPFILE=grafana-npm-${_grafana_version}.tgz
+
+echo "${GCP_KEY}" | base64 -d > credentials.json
+gcloud auth activate-service-account --key-file=credentials.json
+gsutil cp "gs://${PRERELEASE_BUCKET}/artifacts/npm/$ZIPFILE" "$ZIPFILE"
+tar -xzf "$ZIPFILE"

scripts/build/store-npm-packages.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+set -e
+GRAFANA_TAG=${1:-}
+
+if echo "$GRAFANA_TAG" | grep -q "^v"; then
+	_grafana_version=$(echo "${GRAFANA_TAG}" | cut -d "v" -f 2)
+else
+  echo "Provided tag is not a version tag, skipping packages release..."
+	exit
+fi
+
+echo "Storing NPM artifacts"
+ZIPFILE=grafana-npm-${_grafana_version}.tgz
+tar -czf "$ZIPFILE" packages/*/dist packages/*/compiled
+
+echo "${GCP_KEY}" | base64 -d > credentials.json
+gcloud auth activate-service-account --key-file=credentials.json
+gsutil cp "$ZIPFILE" "gs://${PRERELEASE_BUCKET}/artifacts/npm/$ZIPFILE"
+echo "NPM artifacts successfully pushed to GCS."

scripts/drone/pipelines/release.star

@@ -22,7 +22,7 @@ load(
     'e2e_tests_step',
     'build_storybook_step',
     'copy_packages_for_docker_step',
-    'build_docker_images_step',
+    'package_docker_images_step',
     'postgres_integration_tests_step',
     'mysql_integration_tests_step',
     'redis_integration_tests_step',
@@ -51,34 +51,75 @@ load(
     'failure_template',
     'drone_change_template',
 )
+load('scripts/drone/vault.star', 'from_secret', 'github_token', 'pull_secret', 'drone_token', 'prerelease_bucket')
 
-def release_npm_packages_step(edition, ver_mode):
+def build_npm_packages_step(edition, ver_mode):
-    if edition == 'enterprise':
+    if edition == 'enterprise' or ver_mode != 'release':
         return None
 
-    if ver_mode == 'release':
-        commands = ['./scripts/build/release-packages.sh ${DRONE_TAG}']
-    else:
-        commands = []
-
     return {
-        'name': 'release-npm-packages',
+        'name': 'build-npm-packages',
         'image': build_image,
         'depends_on': [
             # Has to run after publish-storybook since this step cleans the files publish-storybook depends on
             'publish-storybook',
         ],
-        'environment': {
+        'commands': ['./scripts/build/build-npm-packages.sh ${DRONE_TAG}'],
-            'NPM_TOKEN': {
-                'from_secret': 'npm_token',
-            },
-            'GITHUB_PACKAGE_TOKEN': {
-                'from_secret': 'github_package_token',
-            },
-        },
-        'commands': commands,
     }
 
+def store_npm_packages_step(edition, ver_mode):
+    if edition == 'enterprise' or ver_mode != 'release':
+        return None
+
+    return {
+        'name': 'store-npm-packages',
+        'image': publish_image,
+        'depends_on': [
+            'build-npm-packages',
+        ],
+        'environment': {
+            'GCP_KEY': from_secret('gcp_key'),
+            'PRERELEASE_BUCKET': from_secret(prerelease_bucket)
+        },
+        'commands': ['./scripts/build/store-npm-packages.sh ${DRONE_TAG}'],
+    }
+
+def retrieve_npm_packages_step(edition, ver_mode):
+    if edition == 'enterprise' or ver_mode != 'release':
+        return None
+
+    return {
+        'name': 'retrieve-npm-packages',
+        'image': publish_image,
+        'depends_on': [
+            # Has to run after publish-storybook since this step cleans the files publish-storybook depends on
+            'publish-storybook',
+        ],
+        'environment': {
+            'GCP_KEY': from_secret('gcp_key'),
+            'PRERELEASE_BUCKET': from_secret(prerelease_bucket)
+        },
+        'commands': ['./scripts/build/retrieve-npm-packages.sh ${DRONE_TAG}'],
+    }
+
+def release_npm_packages_step(edition, ver_mode):
+    if edition == 'enterprise' or ver_mode != 'release':
+        return None
+
+    return {
+        'name': 'release-npm-packages',
+        'image': build_image,
+        'depends_on': [
+            'retrieve-npm-packages',
+        ],
+        'environment': {
+            'NPM_TOKEN': from_secret('npm_token'),
+            'GITHUB_PACKAGE_TOKEN': from_secret('github_package_token'),
+        },
+        'commands': ['./scripts/build/release-npm-packages.sh ${DRONE_TAG}'],
+    }
+
+
 def get_steps(edition, ver_mode):
     build_steps = []
     package_steps = []
@@ -124,8 +165,8 @@ def get_steps(edition, ver_mode):
         e2e_tests_step('panels-suite', edition=edition, tries=3),
         e2e_tests_step('various-suite', edition=edition, tries=3),
         copy_packages_for_docker_step(),
-        build_docker_images_step(edition=edition, ver_mode=ver_mode, publish=should_publish),
+        package_docker_images_step(edition=edition, ver_mode=ver_mode, publish=should_publish),
-        build_docker_images_step(edition=edition, ver_mode=ver_mode, ubuntu=True, publish=should_publish),
+        package_docker_images_step(edition=edition, ver_mode=ver_mode, ubuntu=True, publish=should_publish),
     ])
 
     build_storybook = build_storybook_step(edition=edition, ver_mode=ver_mode)
@@ -140,11 +181,13 @@ def get_steps(edition, ver_mode):
         publish_steps.append(upload_packages_step(edition=edition, ver_mode=ver_mode))
     if should_publish:
         publish_step = publish_storybook_step(edition=edition, ver_mode=ver_mode)
-        release_npm_step = release_npm_packages_step(edition=edition, ver_mode=ver_mode)
+        build_npm_step = build_npm_packages_step(edition=edition, ver_mode=ver_mode)
+        store_npm_step = store_npm_packages_step(edition=edition, ver_mode=ver_mode)
         if publish_step:
             publish_steps.append(publish_step)
-        if release_npm_step:
+        if build_npm_step and store_npm_step:
-            publish_steps.append(release_npm_step)
+            publish_steps.append(build_npm_step)
+            publish_steps.append(store_npm_step)
     windows_package_steps = get_windows_steps(edition=edition, ver_mode=ver_mode)
 
     if include_enterprise2:
@@ -226,7 +269,6 @@ def release_pipelines(ver_mode='release', trigger=None):
             steps=[download_grabpl_step()] + initialize_step(edition='oss', platform='linux', ver_mode=ver_mode, install_deps=False) + steps,
             depends_on=[p['name'] for p in oss_pipelines + enterprise_pipelines],
         )
-        pipelines.append(publish_pipeline)
 
     pipelines.append(notify_pipeline(
         name='notify-{}'.format(ver_mode), slack_channel='grafana-ci-notifications', trigger=dict(trigger, status = ['failure']),

scripts/drone/steps/lib.star

@@ -1,6 +1,6 @@
-load('scripts/drone/vault.star', 'from_secret', 'github_token', 'pull_secret', 'drone_token')
+load('scripts/drone/vault.star', 'from_secret', 'github_token', 'pull_secret', 'drone_token', 'prerelease_bucket')
 
-grabpl_version = '2.7.2'
+grabpl_version = '2.7.4'
 build_image = 'grafana/build-container:1.4.8'
 publish_image = 'grafana/grafana-ci-deploy:1.3.1'
 grafana_docker_image = 'grafana/drone-grafana-docker:0.3.2'
@@ -266,7 +266,7 @@ def publish_storybook_step(edition, ver_mode):
                             'printenv GCP_KEY | base64 -d > /tmp/gcpkey.json',
                             'gcloud auth activate-service-account --key-file=/tmp/gcpkey.json',
                         ] + [
-                            'gsutil -m rsync -d -r ./packages/grafana-ui/dist/storybook gs://grafana-storybook/{}'.format(
+                            'gsutil -m rsync -d -r ./packages/grafana-ui/dist/storybook gs://$${{PRERELEASE_BUCKET}}/artifacts/storybook/{}'.format(
                                 c)
                             for c in channels
                         ])
@@ -283,6 +283,7 @@ def publish_storybook_step(edition, ver_mode):
         ],
         'environment': {
             'GCP_KEY': from_secret('gcp_key'),
+            'PRERELEASE_BUCKET': from_secret(prerelease_bucket)
         },
         'commands': commands,
     }
@@ -297,9 +298,10 @@ def upload_cdn_step(edition):
         ],
         'environment': {
             'GCP_GRAFANA_UPLOAD_KEY': from_secret('gcp_key'),
+            'PRERELEASE_BUCKET': from_secret(prerelease_bucket)
         },
         'commands': [
-            './bin/grabpl upload-cdn --edition {} --bucket "grafana-static-assets"'.format(edition),
+            './bin/grabpl upload-cdn --edition {} --bucket "$${{PRERELEASE_BUCKET}}/artifacts/static-assets"'.format(edition),
         ],
     }
 
@@ -725,6 +727,37 @@ def copy_packages_for_docker_step():
     }
 
 
+def package_docker_images_step(edition, ver_mode, archs=None, ubuntu=False, publish=False):
+    if ver_mode == 'test-release':
+        publish = False
+
+    cmd = './bin/grabpl build-docker --edition {} --shouldSave'.format(edition)
+    ubuntu_sfx = ''
+    if ubuntu:
+        ubuntu_sfx = '-ubuntu'
+        cmd += ' --ubuntu'
+
+    if archs:
+        cmd += ' -archs {}'.format(','.join(archs))
+
+    return {
+        'name': 'package-docker-images' + ubuntu_sfx,
+        'image': 'google/cloud-sdk',
+        'depends_on': ['copy-packages-for-docker'],
+        'commands': [
+            'printenv GCP_KEY | base64 -d > /tmp/gcpkey.json',
+            'gcloud auth activate-service-account --key-file=/tmp/gcpkey.json',
+            cmd
+        ],
+        'volumes': [{
+            'name': 'docker',
+            'path': '/var/run/docker.sock'
+        }],
+        'environment': {
+            'GCP_KEY': from_secret('gcp_key'),
+        },
+    }
+
 def build_docker_images_step(edition, ver_mode, archs=None, ubuntu=False, publish=False):
     if ver_mode == 'test-release':
         publish = False
@@ -878,7 +911,7 @@ def upload_packages_step(edition, ver_mode, is_downstream=False):
     if ver_mode == 'main' and edition in ('enterprise', 'enterprise2') and not is_downstream:
         return None
 
-    packages_bucket = ' --packages-bucket grafana-downloads' + enterprise2_suffix(edition)
+    packages_bucket = ' --packages-bucket $${PRERELEASE_BUCKET}/artifacts/downloads' + enterprise2_suffix(edition)
 
     if ver_mode == 'test-release':
         cmd = './bin/grabpl upload-packages --edition {} '.format(edition) + \
@@ -903,6 +936,7 @@ def upload_packages_step(edition, ver_mode, is_downstream=False):
         'depends_on': dependencies,
         'environment': {
             'GCP_GRAFANA_UPLOAD_KEY': from_secret('gcp_key'),
+            'PRERELEASE_BUCKET': from_secret('prerelease_bucket'),
         },
         'commands': [cmd, ],
     }
@@ -977,7 +1011,7 @@ def get_windows_steps(edition, ver_mode, is_downstream=False):
         'release', 'test-release', 'release-branch',
     ):
         bucket_part = ''
-        bucket = 'grafana-downloads'
+        bucket = '%PRERELEASE_BUCKET%/artifacts/downloads'
         if ver_mode == 'release':
             ver_part = '${DRONE_TAG}'
             dir = 'release'
@@ -1016,6 +1050,7 @@ def get_windows_steps(edition, ver_mode, is_downstream=False):
             'image': wix_image,
             'environment': {
                 'GCP_KEY': from_secret('gcp_key'),
+                'PRERELEASE_BUCKET': from_secret(prerelease_bucket)
             },
             'commands': installer_commands,
             'depends_on': [

scripts/drone/utils/utils.star

@@ -43,6 +43,11 @@ def pipeline(
         'volumes': [{
             'name': 'cypress_cache',
             'temp': {},
+        },{
+            'name': 'docker',
+            'host': {
+                'path': '/var/run/docker.sock',
+            },
         }],
         'depends_on': depends_on,
     }

scripts/drone/vault.star

@@ -1,6 +1,7 @@
 pull_secret = 'dockerconfigjson'
 github_token = 'github_token'
 drone_token = 'drone_token'
+prerelease_bucket = 'prerelease_bucket'
 
 def from_secret(secret):
     return {
@@ -22,4 +23,5 @@ def secrets():
         vault_secret(pull_secret, 'secret/data/common/gcr', '.dockerconfigjson'),
         vault_secret(github_token, 'infra/data/ci/github/grafanabot', 'pat'),
         vault_secret(drone_token, 'infra/data/ci/drone', 'machine-user-token'),
+        vault_secret(prerelease_bucket, 'infra/data/ci/grafana/prerelease', 'bucket'),
     ]