opensource/grafana
1
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-07-30 01:32:08 +08:00
Code Issues Packages Projects Releases Wiki Activity

Package release before publishing (#42218)

Browse Source 
* Package separately to publish

* Fix interpolation

* Windows format envvars

* More descriptive msg

* Won't publish from here

* Resolve docker issues in PR build

* Rename package docker step

* Correct npm release JSON structure
This commit is contained in:
malcolmholmes
2021-11-30 10:53:07 +00:00
committed by GitHub
parent 690ffdff56
commit 9c0a2a19fd
10 changed files with 499 additions and 272 deletions
Download Patch File Download Diff File Expand all files Collapse all files

576
.drone.yml
View File

File diff suppressed because it is too large Load Diff

2
Makefile
View File

@ -158,7 +158,7 @@ gen-ts:
# you modify starlark files.
drone: $(DRONE)
$(DRONE) starlark --format
$(DRONE) lint .drone.yml
$(DRONE) lint .drone.yml --trusted
$(DRONE) --server https://drone.grafana.net sign --save grafana/grafana
help: ## Display this help.

7
scripts/build/build-npm-packages.sh Executable file
View File

@ -0,0 +1,7 @@
#!/usr/bin/env bash
# lerna bootstrap might have created yarn.lock
git checkout .
echo 'Building packages'
yarn packages:build

4
scripts/build/release-packages.sh → scripts/build/release-npm-packages.sh
View File

@ -1,6 +1,7 @@
#!/usr/bin/env bash
PACKAGES=("@grafana/ui" "@grafana/data" "@grafana/toolkit" "@grafana/runtime" "@grafana/e2e" "@grafana/e2e-selectors" "@grafana/schema")
GRAFANA_TAG=${1:-}
RELEASE_CHANNEL="latest"
@ -39,9 +40,6 @@ fi
# Publish to NPM registry
echo "//registry.npmjs.org/:_authToken=${NPM_TOKEN}" >> ~/.npmrc
echo $'\nBuilding packages'
yarn packages:build
echo $'\nPublishing packages to NPM registry'
yarn packages:${SCRIPT}

19
scripts/build/retrieve-npm-packages.sh Executable file
View File

@ -0,0 +1,19 @@
#!/usr/bin/env bash
set -e
GRAFANA_TAG=${1:-}
if echo "$GRAFANA_TAG" | grep -q "^v"; then
_grafana_version=$(echo "${GRAFANA_TAG}" | cut -d "v" -f 2)
else
echo "Provided tag is not a version tag, skipping packages release..."
exit
fi
echo "Retrieving prerelease NPM artifacts"
ZIPFILE=grafana-npm-${_grafana_version}.tgz
echo "${GCP_KEY}" | base64 -d > credentials.json
gcloud auth activate-service-account --key-file=credentials.json
gsutil cp "gs://${PRERELEASE_BUCKET}/artifacts/npm/$ZIPFILE" "$ZIPFILE"
tar -xzf "$ZIPFILE"

19
scripts/build/store-npm-packages.sh Executable file
View File

@ -0,0 +1,19 @@
#!/usr/bin/env bash
set -e
GRAFANA_TAG=${1:-}
if echo "$GRAFANA_TAG" | grep -q "^v"; then
_grafana_version=$(echo "${GRAFANA_TAG}" | cut -d "v" -f 2)
else
echo "Provided tag is not a version tag, skipping packages release..."
exit
fi
echo "Storing NPM artifacts"
ZIPFILE=grafana-npm-${_grafana_version}.tgz
tar -czf "$ZIPFILE" packages/*/dist packages/*/compiled
echo "${GCP_KEY}" | base64 -d > credentials.json
gcloud auth activate-service-account --key-file=credentials.json
gsutil cp "$ZIPFILE" "gs://${PRERELEASE_BUCKET}/artifacts/npm/$ZIPFILE"
echo "NPM artifacts successfully pushed to GCS."

90
scripts/drone/pipelines/release.star
View File

@ -22,7 +22,7 @@ load(
'e2e_tests_step',
'build_storybook_step',
'copy_packages_for_docker_step',
'build_docker_images_step',
'package_docker_images_step',
'postgres_integration_tests_step',
'mysql_integration_tests_step',
'redis_integration_tests_step',
@ -51,34 +51,75 @@ load(
'failure_template',
'drone_change_template',
)
load('scripts/drone/vault.star', 'from_secret', 'github_token', 'pull_secret', 'drone_token', 'prerelease_bucket')
def release_npm_packages_step(edition, ver_mode):
if edition == 'enterprise':
def build_npm_packages_step(edition, ver_mode):
if edition == 'enterprise' or ver_mode != 'release':
return None
if ver_mode == 'release':
commands = ['./scripts/build/release-packages.sh ${DRONE_TAG}']
else:
commands = []
return {
'name': 'release-npm-packages',
'name': 'build-npm-packages',
'image': build_image,
'depends_on': [
# Has to run after publish-storybook since this step cleans the files publish-storybook depends on
'publish-storybook',
],
'environment': {
'NPM_TOKEN': {
'from_secret': 'npm_token',
},
'GITHUB_PACKAGE_TOKEN': {
'from_secret': 'github_package_token',
},
},
'commands': commands,
'commands': ['./scripts/build/build-npm-packages.sh ${DRONE_TAG}'],
}
def store_npm_packages_step(edition, ver_mode):
if edition == 'enterprise' or ver_mode != 'release':
return None
return {
'name': 'store-npm-packages',
'image': publish_image,
'depends_on': [
'build-npm-packages',
],
'environment': {
'GCP_KEY': from_secret('gcp_key'),
'PRERELEASE_BUCKET': from_secret(prerelease_bucket)
},
'commands': ['./scripts/build/store-npm-packages.sh ${DRONE_TAG}'],
}
def retrieve_npm_packages_step(edition, ver_mode):
if edition == 'enterprise' or ver_mode != 'release':
return None
return {
'name': 'retrieve-npm-packages',
'image': publish_image,
'depends_on': [
# Has to run after publish-storybook since this step cleans the files publish-storybook depends on
'publish-storybook',
],
'environment': {
'GCP_KEY': from_secret('gcp_key'),
'PRERELEASE_BUCKET': from_secret(prerelease_bucket)
},
'commands': ['./scripts/build/retrieve-npm-packages.sh ${DRONE_TAG}'],
}
def release_npm_packages_step(edition, ver_mode):
if edition == 'enterprise' or ver_mode != 'release':
return None
return {
'name': 'release-npm-packages',
'image': build_image,
'depends_on': [
'retrieve-npm-packages',
],
'environment': {
'NPM_TOKEN': from_secret('npm_token'),
'GITHUB_PACKAGE_TOKEN': from_secret('github_package_token'),
},
'commands': ['./scripts/build/release-npm-packages.sh ${DRONE_TAG}'],
}
def get_steps(edition, ver_mode):
build_steps = []
package_steps = []
@ -124,8 +165,8 @@ def get_steps(edition, ver_mode):
e2e_tests_step('panels-suite', edition=edition, tries=3),
e2e_tests_step('various-suite', edition=edition, tries=3),
copy_packages_for_docker_step(),
build_docker_images_step(edition=edition, ver_mode=ver_mode, publish=should_publish),
build_docker_images_step(edition=edition, ver_mode=ver_mode, ubuntu=True, publish=should_publish),
package_docker_images_step(edition=edition, ver_mode=ver_mode, publish=should_publish),
package_docker_images_step(edition=edition, ver_mode=ver_mode, ubuntu=True, publish=should_publish),
])
build_storybook = build_storybook_step(edition=edition, ver_mode=ver_mode)
@ -140,11 +181,13 @@ def get_steps(edition, ver_mode):
publish_steps.append(upload_packages_step(edition=edition, ver_mode=ver_mode))
if should_publish:
publish_step = publish_storybook_step(edition=edition, ver_mode=ver_mode)
release_npm_step = release_npm_packages_step(edition=edition, ver_mode=ver_mode)
build_npm_step = build_npm_packages_step(edition=edition, ver_mode=ver_mode)
store_npm_step = store_npm_packages_step(edition=edition, ver_mode=ver_mode)
if publish_step:
publish_steps.append(publish_step)
if release_npm_step:
publish_steps.append(release_npm_step)
if build_npm_step and store_npm_step:
publish_steps.append(build_npm_step)
publish_steps.append(store_npm_step)
windows_package_steps = get_windows_steps(edition=edition, ver_mode=ver_mode)
if include_enterprise2:
@ -226,7 +269,6 @@ def release_pipelines(ver_mode='release', trigger=None):
steps=[download_grabpl_step()] + initialize_step(edition='oss', platform='linux', ver_mode=ver_mode, install_deps=False) + steps,
depends_on=[p['name'] for p in oss_pipelines + enterprise_pipelines],
)
pipelines.append(publish_pipeline)
pipelines.append(notify_pipeline(
name='notify-{}'.format(ver_mode), slack_channel='grafana-ci-notifications', trigger=dict(trigger, status = ['failure']),

47
scripts/drone/steps/lib.star
View File

@ -1,6 +1,6 @@
load('scripts/drone/vault.star', 'from_secret', 'github_token', 'pull_secret', 'drone_token')
load('scripts/drone/vault.star', 'from_secret', 'github_token', 'pull_secret', 'drone_token', 'prerelease_bucket')
grabpl_version = '2.7.2'
grabpl_version = '2.7.4'
build_image = 'grafana/build-container:1.4.8'
publish_image = 'grafana/grafana-ci-deploy:1.3.1'
grafana_docker_image = 'grafana/drone-grafana-docker:0.3.2'
@ -266,7 +266,7 @@ def publish_storybook_step(edition, ver_mode):
'printenv GCP_KEY | base64 -d > /tmp/gcpkey.json',
'gcloud auth activate-service-account --key-file=/tmp/gcpkey.json',
] + [
'gsutil -m rsync -d -r ./packages/grafana-ui/dist/storybook gs://grafana-storybook/{}'.format(
'gsutil -m rsync -d -r ./packages/grafana-ui/dist/storybook gs://$${{PRERELEASE_BUCKET}}/artifacts/storybook/{}'.format(
c)
for c in channels
])
@ -283,6 +283,7 @@ def publish_storybook_step(edition, ver_mode):
],
'environment': {
'GCP_KEY': from_secret('gcp_key'),
'PRERELEASE_BUCKET': from_secret(prerelease_bucket)
},
'commands': commands,
}
@ -297,9 +298,10 @@ def upload_cdn_step(edition):
],
'environment': {
'GCP_GRAFANA_UPLOAD_KEY': from_secret('gcp_key'),
'PRERELEASE_BUCKET': from_secret(prerelease_bucket)
},
'commands': [
'./bin/grabpl upload-cdn --edition {} --bucket "grafana-static-assets"'.format(edition),
'./bin/grabpl upload-cdn --edition {} --bucket "$${{PRERELEASE_BUCKET}}/artifacts/static-assets"'.format(edition),
],
}
@ -725,6 +727,37 @@ def copy_packages_for_docker_step():
}
def package_docker_images_step(edition, ver_mode, archs=None, ubuntu=False, publish=False):
if ver_mode == 'test-release':
publish = False
cmd = './bin/grabpl build-docker --edition {} --shouldSave'.format(edition)
ubuntu_sfx = ''
if ubuntu:
ubuntu_sfx = '-ubuntu'
cmd += ' --ubuntu'
if archs:
cmd += ' -archs {}'.format(','.join(archs))
return {
'name': 'package-docker-images' + ubuntu_sfx,
'image': 'google/cloud-sdk',
'depends_on': ['copy-packages-for-docker'],
'commands': [
'printenv GCP_KEY | base64 -d > /tmp/gcpkey.json',
'gcloud auth activate-service-account --key-file=/tmp/gcpkey.json',
cmd
],
'volumes': [{
'name': 'docker',
'path': '/var/run/docker.sock'
}],
'environment': {
'GCP_KEY': from_secret('gcp_key'),
},
}
def build_docker_images_step(edition, ver_mode, archs=None, ubuntu=False, publish=False):
if ver_mode == 'test-release':
publish = False
@ -878,7 +911,7 @@ def upload_packages_step(edition, ver_mode, is_downstream=False):
if ver_mode == 'main' and edition in ('enterprise', 'enterprise2') and not is_downstream:
return None
packages_bucket = ' --packages-bucket grafana-downloads' + enterprise2_suffix(edition)
packages_bucket = ' --packages-bucket $${PRERELEASE_BUCKET}/artifacts/downloads' + enterprise2_suffix(edition)
if ver_mode == 'test-release':
cmd = './bin/grabpl upload-packages --edition {} '.format(edition) + \
@ -903,6 +936,7 @@ def upload_packages_step(edition, ver_mode, is_downstream=False):
'depends_on': dependencies,
'environment': {
'GCP_GRAFANA_UPLOAD_KEY': from_secret('gcp_key'),
'PRERELEASE_BUCKET': from_secret('prerelease_bucket'),
},
'commands': [cmd, ],
}
@ -977,7 +1011,7 @@ def get_windows_steps(edition, ver_mode, is_downstream=False):
'release', 'test-release', 'release-branch',
):
bucket_part = ''
bucket = 'grafana-downloads'
bucket = '%PRERELEASE_BUCKET%/artifacts/downloads'
if ver_mode == 'release':
ver_part = '${DRONE_TAG}'
dir = 'release'
@ -1016,6 +1050,7 @@ def get_windows_steps(edition, ver_mode, is_downstream=False):
'image': wix_image,
'environment': {
'GCP_KEY': from_secret('gcp_key'),
'PRERELEASE_BUCKET': from_secret(prerelease_bucket)
},
'commands': installer_commands,
'depends_on': [

5
scripts/drone/utils/utils.star
View File

@ -43,6 +43,11 @@ def pipeline(
'volumes': [{
'name': 'cypress_cache',
'temp': {},
},{
'name': 'docker',
'host': {
'path': '/var/run/docker.sock',
},
}],
'depends_on': depends_on,
}

2
scripts/drone/vault.star
View File

@ -1,6 +1,7 @@
pull_secret = 'dockerconfigjson'
github_token = 'github_token'
drone_token = 'drone_token'
prerelease_bucket = 'prerelease_bucket'
def from_secret(secret):
return {
@ -22,4 +23,5 @@ def secrets():
vault_secret(pull_secret, 'secret/data/common/gcr', '.dockerconfigjson'),
vault_secret(github_token, 'infra/data/ci/github/grafanabot', 'pat'),
vault_secret(drone_token, 'infra/data/ci/drone', 'machine-user-token'),
vault_secret(prerelease_bucket, 'infra/data/ci/grafana/prerelease', 'bucket'),
]