opensource/grafana
1
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-07-31 16:02:46 +08:00
Code Issues Packages Projects Releases Wiki Activity

Drone: Fix race conditions between Enterprise and Enterprise2 (#30076)

Browse Source 
* Drone: Fix race conditions between Enterprise and Enterprise2

Signed-off-by: Arve Knudsen <arve.knudsen@gmail.com>
This commit is contained in:
Arve Knudsen
2021-01-06 13:22:20 +01:00
committed by GitHub
parent 2d803ee57e
commit 52f79c3625
5 changed files with 439 additions and 327 deletions
Download Patch File Download Diff File Expand all files Collapse all files

164
scripts/lib.star
View File

@ -1,4 +1,4 @@
grabpl_version = '0.5.30'
grabpl_version = '0.5.31'
build_image = 'grafana/build-container:1.3.0'
publish_image = 'grafana/grafana-ci-deploy:1.2.7'
grafana_docker_image = 'grafana/drone-grafana-docker:0.3.2'
@ -37,7 +37,7 @@ def pipeline(
'depends_on': depends_on,
}
if edition == 'enterprise':
if edition in ('enterprise', 'enterprise2'):
# We have a custom clone step for enterprise
pipeline['clone'] = {
'disable': True,
@ -118,7 +118,7 @@ def init_steps(edition, platform, ver_mode, is_downstream=False, install_deps=Tr
'rm dockerize-linux-amd64-v$${DOCKERIZE_VERSION}.tar.gz',
'yarn install --frozen-lockfile --no-progress',
])
if edition == 'enterprise':
if edition in ('enterprise', 'enterprise2'):
source_commit = ''
if ver_mode == 'release':
committish = '${DRONE_TAG}'
@ -184,7 +184,7 @@ def init_steps(edition, platform, ver_mode, is_downstream=False, install_deps=Tr
return steps
def enterprise_downstream_step(edition):
if edition == 'enterprise':
if edition in ('enterprise', 'enterprise2'):
return None
return {
@ -205,13 +205,10 @@ def enterprise_downstream_step(edition):
},
}
def lint_backend_step(edition, build_tags=None):
def lint_backend_step(edition):
sfx = ''
build_tags_str = ''
if build_tags:
sfx = '-' + '-'.join(build_tags)
build_tags_str += ' --build-tags={}'.format(','.join(build_tags))
if edition == 'enterprise2':
sfx = '-{}'.format(edition)
return {
'name': 'lint-backend' + sfx,
'image': build_image,
@ -224,7 +221,7 @@ def lint_backend_step(edition, build_tags=None):
],
'commands': [
# Don't use Make since it will re-download the linters
'./bin/grabpl lint-backend{}'.format(build_tags_str),
'./bin/grabpl lint-backend --edition {}'.format(edition),
],
}
@ -256,7 +253,7 @@ def ldap_service():
}
def build_storybook_step(edition, ver_mode):
if edition == 'enterprise' and ver_mode in ('release', 'test-release'):
if edition in ('enterprise', 'enterprise2') and ver_mode in ('release', 'test-release'):
return None
return {
@ -276,7 +273,7 @@ def build_storybook_step(edition, ver_mode):
}
def publish_storybook_step(edition, ver_mode):
if edition == 'enterprise':
if edition in ('enterprise', 'enterprise2'):
return None
if ver_mode == 'test-release':
@ -312,16 +309,14 @@ def publish_storybook_step(edition, ver_mode):
'commands': commands,
}
def build_backend_step(edition, ver_mode, variants=None, is_downstream=False, build_tags=None):
def build_backend_step(edition, ver_mode, variants=None, is_downstream=False):
variants_str = ''
if variants:
variants_str = ' --variants {}'.format(','.join(variants))
build_tags_str = ''
sfx = ''
if build_tags:
build_tags_str = ' --build-tags {}'.format(','.join(build_tags))
sfx = '-' + '-'.join(build_tags)
if edition == 'enterprise2':
sfx = '-{}'.format(edition)
# TODO: Convert number of jobs to percentage
if ver_mode == 'release':
@ -331,8 +326,8 @@ def build_backend_step(edition, ver_mode, variants=None, is_downstream=False, bu
},
}
cmds = [
'./bin/grabpl build-backend --jobs 8 --edition {}{} --github-token $${{GITHUB_TOKEN}} --no-pull-enterprise ${{DRONE_TAG}}'.format(
edition, build_tags_str,
'./bin/grabpl build-backend --jobs 8 --edition {} --github-token $${{GITHUB_TOKEN}} --no-pull-enterprise ${{DRONE_TAG}}'.format(
edition,
),
]
elif ver_mode == 'test-release':
@ -342,8 +337,8 @@ def build_backend_step(edition, ver_mode, variants=None, is_downstream=False, bu
},
}
cmds = [
'./bin/grabpl build-backend --jobs 8 --edition {}{} --github-token $${{GITHUB_TOKEN}} --no-pull-enterprise {}'.format(
edition, build_tags_str, test_release_ver,
'./bin/grabpl build-backend --jobs 8 --edition {} --github-token $${{GITHUB_TOKEN}} --no-pull-enterprise {}'.format(
edition, test_release_ver,
),
]
else:
@ -353,8 +348,8 @@ def build_backend_step(edition, ver_mode, variants=None, is_downstream=False, bu
build_no = '$${SOURCE_BUILD_NUMBER}'
env = {}
cmds = [
'./bin/grabpl build-backend --jobs 8 --edition {}{} --build-id {}{} --no-pull-enterprise'.format(
edition, build_tags_str, build_no, variants_str,
'./bin/grabpl build-backend --jobs 8 --edition {} --build-id {}{} --no-pull-enterprise'.format(
edition, build_no, variants_str,
),
]
@ -440,12 +435,10 @@ def build_plugins_step(edition, sign=False):
],
}
def test_backend_step(build_tags=None):
def test_backend_step(edition):
sfx = ''
build_tags_str = ''
if build_tags:
sfx = '-' + '-'.join(build_tags)
build_tags_str = ' --build-tags {}'.format(','.join(build_tags))
if edition == 'enterprise2':
sfx = '-{}'.format(edition)
return {
'name': 'test-backend' + sfx,
@ -458,9 +451,9 @@ def test_backend_step(build_tags=None):
# First make sure that there are no tests with FocusConvey
'[ $(grep FocusConvey -R pkg | wc -l) -eq "0" ] || exit 1',
# Then execute non-integration tests in parallel, since it should be safe
'./bin/grabpl test-backend{}'.format(build_tags_str),
'./bin/grabpl test-backend --edition {}'.format(edition),
# Then execute integration tests in serial
'./bin/grabpl integration-tests{}'.format(build_tags_str),
'./bin/grabpl integration-tests --edition {}'.format(edition),
],
}
@ -480,7 +473,7 @@ def test_frontend_step():
}
def frontend_metrics_step(edition):
if edition == 'enterprise':
if edition in ('enterprise', 'enterprise2'):
return None
return {
@ -539,16 +532,53 @@ def dashboard_schemas_check():
],
}
def package_step(edition, ver_mode, variants=None, is_downstream=False, build_tags=None):
def gen_version_step(ver_mode, include_enterprise2=False, is_downstream=False):
deps = [
'build-backend',
'build-frontend',
'build-plugins',
'test-backend',
'test-frontend',
'codespell',
'shellcheck',
'check-dashboard-schemas',
]
if include_enterprise2:
sfx = '-enterprise2'
deps.extend([
'build-backend' + sfx,
'test-backend' + sfx,
])
if ver_mode == 'release':
args = '${DRONE_TAG}'
elif ver_mode == 'test-release':
args = test_release_ver
else:
if not is_downstream:
build_no = '${DRONE_BUILD_NUMBER}'
else:
build_no = '$${SOURCE_BUILD_NUMBER}'
args = '--build-id {}'.format(build_no)
return {
'name': 'gen-version',
'image': build_image,
'depends_on': deps,
'commands': [
'./bin/grabpl gen-version {}'.format(args),
],
}
def package_step(edition, ver_mode, variants=None, is_downstream=False):
variants_str = ''
if variants:
variants_str = ' --variants {}'.format(','.join(variants))
build_tags_str = ''
sfx = ''
if build_tags:
build_tags_str = ' --build-tags {}'.format(','.join(build_tags))
sfx = '-' + '-'.join(build_tags)
if edition == 'enterprise2':
sfx = '-{}'.format(edition)
if ver_mode in ('master', 'release', 'test-release', 'release-branch'):
sign_args = ' --sign'
@ -578,14 +608,14 @@ def package_step(edition, ver_mode, variants=None, is_downstream=False, build_ta
# TODO: Use percentage for jobs
if ver_mode == 'release':
cmds = [
'{}./bin/grabpl package --jobs 8 --edition {}{} '.format(test_args, edition, build_tags_str) + \
'{}./bin/grabpl package --jobs 8 --edition {} '.format(test_args, edition) + \
'--github-token $${{GITHUB_TOKEN}} --no-pull-enterprise{} ${{DRONE_TAG}}'.format(
sign_args
),
]
elif ver_mode == 'test-release':
cmds = [
'{}./bin/grabpl package --jobs 8 --edition {}{} '.format(test_args, edition, build_tags_str) + \
'{}./bin/grabpl package --jobs 8 --edition {} '.format(test_args, edition) + \
'--github-token $${{GITHUB_TOKEN}} --no-pull-enterprise{} {}'.format(
sign_args, test_release_ver,
),
@ -596,7 +626,7 @@ def package_step(edition, ver_mode, variants=None, is_downstream=False, build_ta
else:
build_no = '$${SOURCE_BUILD_NUMBER}'
cmds = [
'{}./bin/grabpl package --jobs 8 --edition {}{} '.format(test_args, edition, build_tags_str) + \
'{}./bin/grabpl package --jobs 8 --edition {} '.format(test_args, edition) + \
'--build-id {} --no-pull-enterprise{}{}'.format(build_no, variants_str, sign_args),
]
@ -604,24 +634,19 @@ def package_step(edition, ver_mode, variants=None, is_downstream=False, build_ta
'name': 'package' + sfx,
'image': build_image,
'depends_on': [
'build-backend' + sfx,
'build-frontend',
'build-plugins',
'test-backend' + sfx,
'test-frontend',
'codespell',
'shellcheck',
'check-dashboard-schemas',
# This step should have all the dependencies required for packaging, and should generate
# dist/grafana.version
'gen-version',
],
'environment': env,
'commands': cmds,
}
def e2e_tests_server_step(edition, build_tags=None, port=3001):
def e2e_tests_server_step(edition, port=3001):
sfx = ''
package_file_pfx = ''
if build_tags:
sfx = '-' + '-'.join(build_tags)
if edition == 'enterprise2':
sfx = '-{}'.format(edition)
package_file_pfx = 'grafana' + sfx
elif edition == 'enterprise':
package_file_pfx = 'grafana-' + edition
@ -646,10 +671,10 @@ def e2e_tests_server_step(edition, build_tags=None, port=3001):
],
}
def e2e_tests_step(build_tags=None, port=3001):
def e2e_tests_step(edition, port=3001):
sfx = ''
if build_tags:
sfx = '-' + '-'.join(build_tags)
if edition == 'enterprise2':
sfx = '-{}'.format(edition)
return {
'name': 'end-to-end-tests' + sfx,
@ -692,6 +717,7 @@ def copy_packages_for_docker_step():
'package',
],
'commands': [
'ls dist/*.tar.gz*',
'cp dist/*.tar.gz* packaging/docker/',
],
}
@ -775,7 +801,7 @@ def mysql_integration_tests_step():
}
def release_canary_npm_packages_step(edition):
if edition == 'enterprise':
if edition in ('enterprise', 'enterprise2'):
return None
return {
@ -814,23 +840,21 @@ def deploy_to_kubernetes_step(edition, is_downstream=False):
],
}
def upload_packages_step(edition, ver_mode, is_downstream=False, build_tags=None):
if ver_mode == 'master' and edition == 'enterprise' and not is_downstream:
def upload_packages_step(edition, ver_mode, is_downstream=False):
if ver_mode == 'master' and edition in ('enterprise', 'enterprise2') and not is_downstream:
return None
build_tags_str = ''
sfx = ''
packages_bucket = ''
if build_tags:
build_tags_str = ' --build-tags {}'.format(','.join(build_tags))
sfx = '-' + '-'.join(build_tags)
if edition == 'enterprise2':
sfx = '-{}'.format(edition)
packages_bucket = ' --packages-bucket grafana-downloads' + sfx
if ver_mode == 'test-release':
cmd = './bin/grabpl upload-packages --edition {}{} '.format(edition, build_tags_str) + \
cmd = './bin/grabpl upload-packages --edition {} '.format(edition) + \
'--packages-bucket grafana-downloads-test'
else:
cmd = './bin/grabpl upload-packages --edition {}{}{}'.format(edition, build_tags_str, packages_bucket)
cmd = './bin/grabpl upload-packages --edition {}{}'.format(edition, packages_bucket)
return {
'name': 'upload-packages' + sfx,
@ -906,11 +930,11 @@ def get_windows_steps(edition, ver_mode, is_downstream=False):
else:
source_commit = ' $$env:SOURCE_COMMIT'
sfx = ''
if edition == 'enterprise':
sfx = '-enterprise'
init_cmds = []
if edition != 'enterprise':
sfx = ''
if edition in ('enterprise', 'enterprise2'):
sfx = '-{}'.format(edition)
else:
init_cmds.extend([
'$$ProgressPreference = "SilentlyContinue"',
'Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v{}/windows/grabpl.exe -OutFile grabpl.exe'.format(grabpl_version),
@ -923,7 +947,7 @@ def get_windows_steps(edition, ver_mode, is_downstream=False):
'commands': init_cmds,
},
]
if (ver_mode == 'master' and (edition != 'enterprise' or is_downstream)) or ver_mode in (
if (ver_mode == 'master' and (edition not in ('enterprise', 'enterprise2') or is_downstream)) or ver_mode in (
'release', 'test-release', 'release-branch',
):
bucket_part = ''
@ -952,7 +976,7 @@ def get_windows_steps(edition, ver_mode, is_downstream=False):
'rm gcpkey.json',
'cp C:\\App\\nssm-2.24.zip .',
]
if (ver_mode == 'master' and (edition != 'enterprise' or is_downstream)) or ver_mode in (
if (ver_mode == 'master' and (edition not in ('enterprise', 'enterprise2') or is_downstream)) or ver_mode in (
'release', 'test-release',
):
installer_commands.extend([
@ -975,7 +999,7 @@ def get_windows_steps(edition, ver_mode, is_downstream=False):
],
})
if edition == 'enterprise':
if edition in ('enterprise', 'enterprise2'):
if ver_mode == 'release':
committish = '${DRONE_TAG}'
elif ver_mode == 'test-release':

37
scripts/master.star
View File

@ -10,6 +10,7 @@ load(
'build_backend_step',
'build_frontend_step',
'build_plugins_step',
'gen_version_step',
'package_step',
'e2e_tests_server_step',
'e2e_tests_step',
@ -37,21 +38,36 @@ ver_mode = 'master'
def get_steps(edition, is_downstream=False):
publish = edition != 'enterprise' or is_downstream
include_enterprise2 = edition == 'enterprise'
steps = [
enterprise_downstream_step(edition=edition),
lint_backend_step(edition=edition),
codespell_step(),
shellcheck_step(),
dashboard_schemas_check(),
test_backend_step(),
test_backend_step(edition=edition),
test_frontend_step(),
frontend_metrics_step(edition=edition),
build_backend_step(edition=edition, ver_mode=ver_mode, is_downstream=is_downstream),
build_frontend_step(edition=edition, ver_mode=ver_mode, is_downstream=is_downstream),
build_plugins_step(edition=edition, sign=True),
]
# Have to insert Enterprise2 steps before they're depended on (in the gen-version step)
if include_enterprise2:
edition2 = 'enterprise2'
steps.extend([
lint_backend_step(edition=edition2),
test_backend_step(edition=edition2),
build_backend_step(edition=edition2, ver_mode=ver_mode, variants=['linux-x64'], is_downstream=is_downstream),
])
# Insert remaining steps
steps.extend([
gen_version_step(ver_mode=ver_mode, is_downstream=is_downstream, include_enterprise2=include_enterprise2),
package_step(edition=edition, ver_mode=ver_mode, is_downstream=is_downstream),
e2e_tests_server_step(edition=edition),
e2e_tests_step(),
e2e_tests_step(edition=edition),
build_storybook_step(edition=edition, ver_mode=ver_mode),
publish_storybook_step(edition=edition, ver_mode=ver_mode),
build_frontend_docs_step(edition=edition),
@ -63,17 +79,14 @@ def get_steps(edition, is_downstream=False):
release_canary_npm_packages_step(edition),
upload_packages_step(edition=edition, ver_mode=ver_mode, is_downstream=is_downstream),
deploy_to_kubernetes_step(edition=edition, is_downstream=is_downstream),
]
if edition == 'enterprise':
build_tags = ['enterprise2']
])
if include_enterprise2:
edition2 = 'enterprise2'
steps.extend([
lint_backend_step(edition=edition, build_tags=build_tags),
test_backend_step(build_tags=build_tags),
build_backend_step(edition=edition, ver_mode=ver_mode, variants=['linux-x64'], is_downstream=is_downstream, build_tags=build_tags),
package_step(edition=edition, ver_mode=ver_mode, variants=['linux-x64'], is_downstream=is_downstream, build_tags=build_tags),
e2e_tests_server_step(edition=edition, build_tags=build_tags, port=3002),
e2e_tests_step(build_tags=build_tags, port=3002),
upload_packages_step(edition=edition, ver_mode=ver_mode, is_downstream=is_downstream, build_tags=build_tags),
package_step(edition=edition2, ver_mode=ver_mode, variants=['linux-x64'], is_downstream=is_downstream),
e2e_tests_server_step(edition=edition2, port=3002),
e2e_tests_step(edition=edition2, port=3002),
upload_packages_step(edition=edition2, ver_mode=ver_mode, is_downstream=is_downstream),
])
windows_steps = get_windows_steps(edition=edition, ver_mode=ver_mode, is_downstream=is_downstream)

39
scripts/pr.star
View File

@ -10,6 +10,7 @@ load(
'build_backend_step',
'build_frontend_step',
'build_plugins_step',
'gen_version_step',
'package_step',
'e2e_tests_server_step',
'e2e_tests_step',
@ -30,19 +31,36 @@ ver_mode = 'pr'
def pr_pipelines(edition):
services = integration_test_services()
variants = ['linux-x64', 'linux-x64-musl', 'osx64', 'win64',]
include_enterprise2 = edition == 'enterprise'
steps = [
lint_backend_step(edition=edition),
codespell_step(),
shellcheck_step(),
dashboard_schemas_check(),
test_backend_step(),
test_backend_step(edition=edition),
test_frontend_step(),
build_backend_step(edition=edition, ver_mode=ver_mode, variants=variants),
build_frontend_step(edition=edition, ver_mode=ver_mode),
build_plugins_step(edition=edition),
]
# Have to insert Enterprise2 steps before they're depended on (in the gen-version step)
if include_enterprise2:
edition2 = 'enterprise2'
steps.append(benchmark_ldap_step())
services.append(ldap_service())
steps.extend([
lint_backend_step(edition=edition2),
test_backend_step(edition=edition2),
build_backend_step(edition=edition2, ver_mode=ver_mode, variants=['linux-x64']),
])
# Insert remaining steps
steps.extend([
gen_version_step(ver_mode=ver_mode, include_enterprise2=include_enterprise2),
package_step(edition=edition, ver_mode=ver_mode, variants=variants),
e2e_tests_server_step(edition=edition),
e2e_tests_step(),
e2e_tests_step(edition=edition),
build_storybook_step(edition=edition, ver_mode=ver_mode),
build_frontend_docs_step(edition=edition),
build_docs_website_step(),
@ -50,18 +68,13 @@ def pr_pipelines(edition):
build_docker_images_step(edition=edition, ver_mode=ver_mode, archs=['amd64',]),
postgres_integration_tests_step(),
mysql_integration_tests_step(),
]
if edition == 'enterprise':
steps.append(benchmark_ldap_step())
services.append(ldap_service())
build_tags = ['enterprise2']
])
if include_enterprise2:
steps.extend([
lint_backend_step(edition=edition, build_tags=build_tags),
test_backend_step(build_tags=build_tags),
build_backend_step(edition=edition, ver_mode=ver_mode, variants=['linux-x64'], build_tags=build_tags),
package_step(edition=edition, ver_mode=ver_mode, variants=['linux-x64'], build_tags=build_tags),
e2e_tests_server_step(edition=edition, build_tags=build_tags, port=3002),
e2e_tests_step(build_tags=build_tags, port=3002),
package_step(edition=edition2, ver_mode=ver_mode, variants=['linux-x64']),
e2e_tests_server_step(edition=edition2, port=3002),
e2e_tests_step(edition=edition2, port=3002),
])
trigger = {

37
scripts/release.star
View File

@ -13,6 +13,7 @@ load(
'build_backend_step',
'build_frontend_step',
'build_plugins_step',
'gen_version_step',
'package_step',
'e2e_tests_server_step',
'e2e_tests_step',
@ -59,27 +60,42 @@ def release_npm_packages_step(edition, ver_mode):
def get_steps(edition, ver_mode):
should_publish = ver_mode in ('release', 'test-release',)
should_upload = should_publish or ver_mode in ('release-branch',)
include_enterprise2 = edition == 'enterprise'
steps = [
lint_backend_step(edition=edition),
codespell_step(),
shellcheck_step(),
dashboard_schemas_check(),
test_backend_step(),
test_backend_step(edition=edition),
test_frontend_step(),
build_backend_step(edition=edition, ver_mode=ver_mode),
build_frontend_step(edition=edition, ver_mode=ver_mode),
build_plugins_step(edition=edition, sign=True),
]
# Have to insert Enterprise2 steps before they're depended on (in the gen-version step)
if include_enterprise2:
edition2 = 'enterprise2'
steps.extend([
lint_backend_step(edition=edition2),
test_backend_step(edition=edition2),
build_backend_step(edition=edition2, ver_mode=ver_mode, variants=['linux-x64']),
])
# Insert remaining steps
steps.extend([
gen_version_step(ver_mode=ver_mode, include_enterprise2=include_enterprise2),
package_step(edition=edition, ver_mode=ver_mode),
e2e_tests_server_step(edition=edition),
e2e_tests_step(),
e2e_tests_step(edition=edition),
build_storybook_step(edition=edition, ver_mode=ver_mode),
copy_packages_for_docker_step(),
build_docker_images_step(edition=edition, ver_mode=ver_mode, publish=should_publish),
build_docker_images_step(edition=edition, ver_mode=ver_mode, ubuntu=True, publish=should_publish),
postgres_integration_tests_step(),
mysql_integration_tests_step(),
]
])
if should_upload:
steps.append(upload_packages_step(edition=edition, ver_mode=ver_mode))
if should_publish:
@ -89,18 +105,15 @@ def get_steps(edition, ver_mode):
])
windows_steps = get_windows_steps(edition=edition, ver_mode=ver_mode)
if edition == 'enterprise':
build_tags = ['enterprise2']
if include_enterprise2:
edition2 = 'enterprise2'
steps.extend([
lint_backend_step(edition=edition, build_tags=build_tags),
test_backend_step(build_tags=build_tags),
build_backend_step(edition=edition, ver_mode=ver_mode, variants=['linux-x64'], build_tags=build_tags),
package_step(edition=edition, ver_mode=ver_mode, variants=['linux-x64'], build_tags=build_tags),
e2e_tests_server_step(edition=edition, build_tags=build_tags, port=3002),
e2e_tests_step(build_tags=build_tags, port=3002),
package_step(edition=edition2, ver_mode=ver_mode, variants=['linux-x64']),
e2e_tests_server_step(edition=edition2, port=3002),
e2e_tests_step(edition=edition2, port=3002),
])
if should_upload:
steps.append(upload_packages_step(edition=edition, ver_mode=ver_mode, build_tags=build_tags))
steps.append(upload_packages_step(edition=edition2, ver_mode=ver_mode))
return steps, windows_steps