opensource/grafana
1
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-07-29 16:12:14 +08:00
Code Issues Packages Projects Releases Wiki Activity

Package release before publishing (#42218)

Browse Source 
* Package separately to publish

* Fix interpolation

* Windows format envvars

* More descriptive msg

* Won't publish from here

* Resolve docker issues in PR build

* Rename package docker step

* Correct npm release JSON structure
This commit is contained in:
malcolmholmes
2021-11-30 10:53:07 +00:00
committed by GitHub
parent 690ffdff56
commit 9c0a2a19fd
10 changed files with 499 additions and 272 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
scripts/build/build-npm-packages.sh Executable file
View File

@ -0,0 +1,7 @@
#!/usr/bin/env bash
# lerna bootstrap might have created yarn.lock
git checkout .
echo 'Building packages'
yarn packages:build

4
scripts/build/release-packages.sh → scripts/build/release-npm-packages.sh
View File

@ -1,6 +1,7 @@
#!/usr/bin/env bash
PACKAGES=("@grafana/ui" "@grafana/data" "@grafana/toolkit" "@grafana/runtime" "@grafana/e2e" "@grafana/e2e-selectors" "@grafana/schema")
GRAFANA_TAG=${1:-}
RELEASE_CHANNEL="latest"
@ -39,9 +40,6 @@ fi
# Publish to NPM registry
echo "//registry.npmjs.org/:_authToken=${NPM_TOKEN}" >> ~/.npmrc
echo $'\nBuilding packages'
yarn packages:build
echo $'\nPublishing packages to NPM registry'
yarn packages:${SCRIPT}

19
scripts/build/retrieve-npm-packages.sh Executable file
View File

@ -0,0 +1,19 @@
#!/usr/bin/env bash
set -e
GRAFANA_TAG=${1:-}
if echo "$GRAFANA_TAG" | grep -q "^v"; then
_grafana_version=$(echo "${GRAFANA_TAG}" | cut -d "v" -f 2)
else
echo "Provided tag is not a version tag, skipping packages release..."
exit
fi
echo "Retrieving prerelease NPM artifacts"
ZIPFILE=grafana-npm-${_grafana_version}.tgz
echo "${GCP_KEY}" | base64 -d > credentials.json
gcloud auth activate-service-account --key-file=credentials.json
gsutil cp "gs://${PRERELEASE_BUCKET}/artifacts/npm/$ZIPFILE" "$ZIPFILE"
tar -xzf "$ZIPFILE"

19
scripts/build/store-npm-packages.sh Executable file
View File

@ -0,0 +1,19 @@
#!/usr/bin/env bash
set -e
GRAFANA_TAG=${1:-}
if echo "$GRAFANA_TAG" | grep -q "^v"; then
_grafana_version=$(echo "${GRAFANA_TAG}" | cut -d "v" -f 2)
else
echo "Provided tag is not a version tag, skipping packages release..."
exit
fi
echo "Storing NPM artifacts"
ZIPFILE=grafana-npm-${_grafana_version}.tgz
tar -czf "$ZIPFILE" packages/*/dist packages/*/compiled
echo "${GCP_KEY}" | base64 -d > credentials.json
gcloud auth activate-service-account --key-file=credentials.json
gsutil cp "$ZIPFILE" "gs://${PRERELEASE_BUCKET}/artifacts/npm/$ZIPFILE"
echo "NPM artifacts successfully pushed to GCS."

90
scripts/drone/pipelines/release.star
View File

@ -22,7 +22,7 @@ load(
'e2e_tests_step',
'build_storybook_step',
'copy_packages_for_docker_step',
'build_docker_images_step',
'package_docker_images_step',
'postgres_integration_tests_step',
'mysql_integration_tests_step',
'redis_integration_tests_step',
@ -51,34 +51,75 @@ load(
'failure_template',
'drone_change_template',
)
load('scripts/drone/vault.star', 'from_secret', 'github_token', 'pull_secret', 'drone_token', 'prerelease_bucket')
def release_npm_packages_step(edition, ver_mode):
if edition == 'enterprise':
def build_npm_packages_step(edition, ver_mode):
if edition == 'enterprise' or ver_mode != 'release':
return None
if ver_mode == 'release':
commands = ['./scripts/build/release-packages.sh ${DRONE_TAG}']
else:
commands = []
return {
'name': 'release-npm-packages',
'name': 'build-npm-packages',
'image': build_image,
'depends_on': [
# Has to run after publish-storybook since this step cleans the files publish-storybook depends on
'publish-storybook',
],
'environment': {
'NPM_TOKEN': {
'from_secret': 'npm_token',
},
'GITHUB_PACKAGE_TOKEN': {
'from_secret': 'github_package_token',
},
},
'commands': commands,
'commands': ['./scripts/build/build-npm-packages.sh ${DRONE_TAG}'],
}
def store_npm_packages_step(edition, ver_mode):
if edition == 'enterprise' or ver_mode != 'release':
return None
return {
'name': 'store-npm-packages',
'image': publish_image,
'depends_on': [
'build-npm-packages',
],
'environment': {
'GCP_KEY': from_secret('gcp_key'),
'PRERELEASE_BUCKET': from_secret(prerelease_bucket)
},
'commands': ['./scripts/build/store-npm-packages.sh ${DRONE_TAG}'],
}
def retrieve_npm_packages_step(edition, ver_mode):
if edition == 'enterprise' or ver_mode != 'release':
return None
return {
'name': 'retrieve-npm-packages',
'image': publish_image,
'depends_on': [
# Has to run after publish-storybook since this step cleans the files publish-storybook depends on
'publish-storybook',
],
'environment': {
'GCP_KEY': from_secret('gcp_key'),
'PRERELEASE_BUCKET': from_secret(prerelease_bucket)
},
'commands': ['./scripts/build/retrieve-npm-packages.sh ${DRONE_TAG}'],
}
def release_npm_packages_step(edition, ver_mode):
if edition == 'enterprise' or ver_mode != 'release':
return None
return {
'name': 'release-npm-packages',
'image': build_image,
'depends_on': [
'retrieve-npm-packages',
],
'environment': {
'NPM_TOKEN': from_secret('npm_token'),
'GITHUB_PACKAGE_TOKEN': from_secret('github_package_token'),
},
'commands': ['./scripts/build/release-npm-packages.sh ${DRONE_TAG}'],
}
def get_steps(edition, ver_mode):
build_steps = []
package_steps = []
@ -124,8 +165,8 @@ def get_steps(edition, ver_mode):
e2e_tests_step('panels-suite', edition=edition, tries=3),
e2e_tests_step('various-suite', edition=edition, tries=3),
copy_packages_for_docker_step(),
build_docker_images_step(edition=edition, ver_mode=ver_mode, publish=should_publish),
build_docker_images_step(edition=edition, ver_mode=ver_mode, ubuntu=True, publish=should_publish),
package_docker_images_step(edition=edition, ver_mode=ver_mode, publish=should_publish),
package_docker_images_step(edition=edition, ver_mode=ver_mode, ubuntu=True, publish=should_publish),
])
build_storybook = build_storybook_step(edition=edition, ver_mode=ver_mode)
@ -140,11 +181,13 @@ def get_steps(edition, ver_mode):
publish_steps.append(upload_packages_step(edition=edition, ver_mode=ver_mode))
if should_publish:
publish_step = publish_storybook_step(edition=edition, ver_mode=ver_mode)
release_npm_step = release_npm_packages_step(edition=edition, ver_mode=ver_mode)
build_npm_step = build_npm_packages_step(edition=edition, ver_mode=ver_mode)
store_npm_step = store_npm_packages_step(edition=edition, ver_mode=ver_mode)
if publish_step:
publish_steps.append(publish_step)
if release_npm_step:
publish_steps.append(release_npm_step)
if build_npm_step and store_npm_step:
publish_steps.append(build_npm_step)
publish_steps.append(store_npm_step)
windows_package_steps = get_windows_steps(edition=edition, ver_mode=ver_mode)
if include_enterprise2:
@ -226,7 +269,6 @@ def release_pipelines(ver_mode='release', trigger=None):
steps=[download_grabpl_step()] + initialize_step(edition='oss', platform='linux', ver_mode=ver_mode, install_deps=False) + steps,
depends_on=[p['name'] for p in oss_pipelines + enterprise_pipelines],
)
pipelines.append(publish_pipeline)
pipelines.append(notify_pipeline(
name='notify-{}'.format(ver_mode), slack_channel='grafana-ci-notifications', trigger=dict(trigger, status = ['failure']),

47
scripts/drone/steps/lib.star
View File

@ -1,6 +1,6 @@
load('scripts/drone/vault.star', 'from_secret', 'github_token', 'pull_secret', 'drone_token')
load('scripts/drone/vault.star', 'from_secret', 'github_token', 'pull_secret', 'drone_token', 'prerelease_bucket')
grabpl_version = '2.7.2'
grabpl_version = '2.7.4'
build_image = 'grafana/build-container:1.4.8'
publish_image = 'grafana/grafana-ci-deploy:1.3.1'
grafana_docker_image = 'grafana/drone-grafana-docker:0.3.2'
@ -266,7 +266,7 @@ def publish_storybook_step(edition, ver_mode):
'printenv GCP_KEY | base64 -d > /tmp/gcpkey.json',
'gcloud auth activate-service-account --key-file=/tmp/gcpkey.json',
] + [
'gsutil -m rsync -d -r ./packages/grafana-ui/dist/storybook gs://grafana-storybook/{}'.format(
'gsutil -m rsync -d -r ./packages/grafana-ui/dist/storybook gs://$${{PRERELEASE_BUCKET}}/artifacts/storybook/{}'.format(
c)
for c in channels
])
@ -283,6 +283,7 @@ def publish_storybook_step(edition, ver_mode):
],
'environment': {
'GCP_KEY': from_secret('gcp_key'),
'PRERELEASE_BUCKET': from_secret(prerelease_bucket)
},
'commands': commands,
}
@ -297,9 +298,10 @@ def upload_cdn_step(edition):
],
'environment': {
'GCP_GRAFANA_UPLOAD_KEY': from_secret('gcp_key'),
'PRERELEASE_BUCKET': from_secret(prerelease_bucket)
},
'commands': [
'./bin/grabpl upload-cdn --edition {} --bucket "grafana-static-assets"'.format(edition),
'./bin/grabpl upload-cdn --edition {} --bucket "$${{PRERELEASE_BUCKET}}/artifacts/static-assets"'.format(edition),
],
}
@ -725,6 +727,37 @@ def copy_packages_for_docker_step():
}
def package_docker_images_step(edition, ver_mode, archs=None, ubuntu=False, publish=False):
if ver_mode == 'test-release':
publish = False
cmd = './bin/grabpl build-docker --edition {} --shouldSave'.format(edition)
ubuntu_sfx = ''
if ubuntu:
ubuntu_sfx = '-ubuntu'
cmd += ' --ubuntu'
if archs:
cmd += ' -archs {}'.format(','.join(archs))
return {
'name': 'package-docker-images' + ubuntu_sfx,
'image': 'google/cloud-sdk',
'depends_on': ['copy-packages-for-docker'],
'commands': [
'printenv GCP_KEY | base64 -d > /tmp/gcpkey.json',
'gcloud auth activate-service-account --key-file=/tmp/gcpkey.json',
cmd
],
'volumes': [{
'name': 'docker',
'path': '/var/run/docker.sock'
}],
'environment': {
'GCP_KEY': from_secret('gcp_key'),
},
}
def build_docker_images_step(edition, ver_mode, archs=None, ubuntu=False, publish=False):
if ver_mode == 'test-release':
publish = False
@ -878,7 +911,7 @@ def upload_packages_step(edition, ver_mode, is_downstream=False):
if ver_mode == 'main' and edition in ('enterprise', 'enterprise2') and not is_downstream:
return None
packages_bucket = ' --packages-bucket grafana-downloads' + enterprise2_suffix(edition)
packages_bucket = ' --packages-bucket $${PRERELEASE_BUCKET}/artifacts/downloads' + enterprise2_suffix(edition)
if ver_mode == 'test-release':
cmd = './bin/grabpl upload-packages --edition {} '.format(edition) + \
@ -903,6 +936,7 @@ def upload_packages_step(edition, ver_mode, is_downstream=False):
'depends_on': dependencies,
'environment': {
'GCP_GRAFANA_UPLOAD_KEY': from_secret('gcp_key'),
'PRERELEASE_BUCKET': from_secret('prerelease_bucket'),
},
'commands': [cmd, ],
}
@ -977,7 +1011,7 @@ def get_windows_steps(edition, ver_mode, is_downstream=False):
'release', 'test-release', 'release-branch',
):
bucket_part = ''
bucket = 'grafana-downloads'
bucket = '%PRERELEASE_BUCKET%/artifacts/downloads'
if ver_mode == 'release':
ver_part = '${DRONE_TAG}'
dir = 'release'
@ -1016,6 +1050,7 @@ def get_windows_steps(edition, ver_mode, is_downstream=False):
'image': wix_image,
'environment': {
'GCP_KEY': from_secret('gcp_key'),
'PRERELEASE_BUCKET': from_secret(prerelease_bucket)
},
'commands': installer_commands,
'depends_on': [

5
scripts/drone/utils/utils.star
View File

@ -43,6 +43,11 @@ def pipeline(
'volumes': [{
'name': 'cypress_cache',
'temp': {},
},{
'name': 'docker',
'host': {
'path': '/var/run/docker.sock',
},
}],
'depends_on': depends_on,
}

2
scripts/drone/vault.star
View File

@ -1,6 +1,7 @@
pull_secret = 'dockerconfigjson'
github_token = 'github_token'
drone_token = 'drone_token'
prerelease_bucket = 'prerelease_bucket'
def from_secret(secret):
return {
@ -22,4 +23,5 @@ def secrets():
vault_secret(pull_secret, 'secret/data/common/gcr', '.dockerconfigjson'),
vault_secret(github_token, 'infra/data/ci/github/grafanabot', 'pat'),
vault_secret(drone_token, 'infra/data/ci/drone', 'machine-user-token'),
vault_secret(prerelease_bucket, 'infra/data/ci/grafana/prerelease', 'bucket'),
]