From 2cda971796fb23f64575b139b74dd3cd46dc5635 Mon Sep 17 00:00:00 2001 From: Dimitris Sotirakis Date: Mon, 12 Jun 2023 16:41:18 +0300 Subject: [PATCH] Security Scans: Add trivy scans to every docker image used for building/testing/publishing (#69911) * Created images.star * Fix typo * Add cronjobs for build-images --- .drone.yml | 84 ++++++++++- scripts/drone/events/cron.star | 73 ++++++++-- scripts/drone/events/release.star | 21 +-- scripts/drone/pipelines/aws_marketplace.star | 7 +- scripts/drone/pipelines/ci_images.star | 12 +- scripts/drone/pipelines/docs.star | 7 +- scripts/drone/pipelines/github.star | 7 +- scripts/drone/pipelines/shellcheck.star | 8 +- scripts/drone/pipelines/windows.star | 7 +- scripts/drone/services/services.star | 17 ++- scripts/drone/steps/lib.star | 142 +++++++++---------- scripts/drone/utils/images.star | 29 ++++ 12 files changed, 290 insertions(+), 124 deletions(-) create mode 100644 scripts/drone/utils/images.star diff --git a/.drone.yml b/.drone.yml index 3352004c43d..35c6be52526 100644 --- a/.drone.yml +++ b/.drone.yml @@ -6925,7 +6925,7 @@ steps: - commands: - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/grafana:latest image: aquasec/trivy:0.21.0 - name: scan-unkown-low-medium-vulnerabilities + name: scan-unknown-low-medium-vulnerabilities - commands: - trivy --exit-code 1 --severity HIGH,CRITICAL grafana/grafana:latest image: aquasec/trivy:0.21.0 @@ -6955,7 +6955,7 @@ steps: - commands: - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/grafana:main image: aquasec/trivy:0.21.0 - name: scan-unkown-low-medium-vulnerabilities + name: scan-unknown-low-medium-vulnerabilities - commands: - trivy --exit-code 1 --severity HIGH,CRITICAL grafana/grafana:main image: aquasec/trivy:0.21.0 @@ -6985,7 +6985,7 @@ steps: - commands: - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/grafana:latest-ubuntu image: aquasec/trivy:0.21.0 - name: scan-unkown-low-medium-vulnerabilities + name: scan-unknown-low-medium-vulnerabilities - commands: - trivy --exit-code 1 --severity HIGH,CRITICAL grafana/grafana:latest-ubuntu image: aquasec/trivy:0.21.0 @@ -7016,7 +7016,7 @@ steps: - commands: - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/grafana:main-ubuntu image: aquasec/trivy:0.21.0 - name: scan-unkown-low-medium-vulnerabilities + name: scan-unknown-low-medium-vulnerabilities - commands: - trivy --exit-code 1 --severity HIGH,CRITICAL grafana/grafana:main-ubuntu image: aquasec/trivy:0.21.0 @@ -7039,6 +7039,80 @@ type: docker clone: retries: 3 kind: pipeline +name: scan-build-test-and-publish-docker-images +platform: + arch: amd64 + os: linux +steps: +- commands: + - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM google/cloud-sdk:431.0.0 + - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/build-container:1.7.4 + - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/grafana-ci-deploy:1.3.3 + - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM alpine:3.17.1 + - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM byrnedo/alpine-curl:0.1.8 + - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM mcr.microsoft.com/windows:1809 + - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/ci-wix:0.1.1 + - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM golang:1.20.4 + - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/grafana-ci-windows-test:0.1.0 + - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM plugins/slack + - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM postgres:12.3-alpine + - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM mysql:5.7.39 + - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM mysql:8.0.32 + - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM redis:6.2.11-alpine + - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM memcached:1.6.9-alpine + - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM docker:windowsservercore-1809 + - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM us.gcr.io/kubernetes-dev/package-publish:latest + - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM osixia/openldap:1.4.0 + - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/drone-downstream + - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/docker-puppeteer:1.1.0 + - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/docs-base:latest + - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97 + - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM us-docker.pkg.dev/grafanalabs-dev/cloud-data-sources/e2e:latest + image: aquasec/trivy:0.21.0 + name: scan-unknown-low-medium-vulnerabilities +- commands: + - trivy --exit-code 1 --severity HIGH,CRITICAL google/cloud-sdk:431.0.0 + - trivy --exit-code 1 --severity HIGH,CRITICAL grafana/build-container:1.7.4 + - trivy --exit-code 1 --severity HIGH,CRITICAL grafana/grafana-ci-deploy:1.3.3 + - trivy --exit-code 1 --severity HIGH,CRITICAL alpine:3.17.1 + - trivy --exit-code 1 --severity HIGH,CRITICAL byrnedo/alpine-curl:0.1.8 + - trivy --exit-code 1 --severity HIGH,CRITICAL mcr.microsoft.com/windows:1809 + - trivy --exit-code 1 --severity HIGH,CRITICAL grafana/ci-wix:0.1.1 + - trivy --exit-code 1 --severity HIGH,CRITICAL golang:1.20.4 + - trivy --exit-code 1 --severity HIGH,CRITICAL grafana/grafana-ci-windows-test:0.1.0 + - trivy --exit-code 1 --severity HIGH,CRITICAL plugins/slack + - trivy --exit-code 1 --severity HIGH,CRITICAL postgres:12.3-alpine + - trivy --exit-code 1 --severity HIGH,CRITICAL mysql:5.7.39 + - trivy --exit-code 1 --severity HIGH,CRITICAL mysql:8.0.32 + - trivy --exit-code 1 --severity HIGH,CRITICAL redis:6.2.11-alpine + - trivy --exit-code 1 --severity HIGH,CRITICAL memcached:1.6.9-alpine + - trivy --exit-code 1 --severity HIGH,CRITICAL docker:windowsservercore-1809 + - trivy --exit-code 1 --severity HIGH,CRITICAL us.gcr.io/kubernetes-dev/package-publish:latest + - trivy --exit-code 1 --severity HIGH,CRITICAL osixia/openldap:1.4.0 + - trivy --exit-code 1 --severity HIGH,CRITICAL grafana/drone-downstream + - trivy --exit-code 1 --severity HIGH,CRITICAL grafana/docker-puppeteer:1.1.0 + - trivy --exit-code 1 --severity HIGH,CRITICAL grafana/docs-base:latest + - trivy --exit-code 1 --severity HIGH,CRITICAL cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97 + - trivy --exit-code 1 --severity HIGH,CRITICAL us-docker.pkg.dev/grafanalabs-dev/cloud-data-sources/e2e:latest + image: aquasec/trivy:0.21.0 + name: scan-high-critical-vulnerabilities +- image: plugins/slack + name: slack-notify-failure + settings: + channel: grafana-backend-ops + template: 'Nightly docker image scan job for build-images failed: {{build.link}}' + webhook: + from_secret: slack_webhook_backend + when: + status: failure +trigger: + cron: nightly + event: cron +type: docker +--- +clone: + retries: 3 +kind: pipeline name: grafana-com-nightly platform: arch: amd64 @@ -7242,6 +7316,6 @@ kind: secret name: delivery-bot-app-private-key --- kind: signature -hmac: caf02d8ce78dd5c2fd6d841d0661702e74571c07939c6af602ac11d1790b31b8 +hmac: 44f95e0e3d9eb3fc8891e94f7205c18e85adacab25b69906d3c5212875baa383 ... diff --git a/scripts/drone/events/cron.star b/scripts/drone/events/cron.star index 9b718f79572..d73e1074562 100644 --- a/scripts/drone/events/cron.star +++ b/scripts/drone/events/cron.star @@ -6,7 +6,10 @@ load("scripts/drone/vault.star", "from_secret") load( "scripts/drone/steps/lib.star", "compile_build_cmd", - "publish_image", +) +load( + "scripts/drone/utils/images.star", + "images", ) aquasec_trivy_image = "aquasec/trivy:0.21.0" @@ -17,6 +20,7 @@ def cronjobs(): scan_docker_image_pipeline("main"), scan_docker_image_pipeline("latest-ubuntu"), scan_docker_image_pipeline("main-ubuntu"), + scan_build_test_publish_docker_image_pipeline(), grafana_com_nightly_pipeline(), ] @@ -54,34 +58,77 @@ def scan_docker_image_pipeline(tag): cronName = "nightly", name = "scan-" + docker_image + "-image", steps = [ - scan_docker_image_unkown_low_medium_vulnerabilities_step(docker_image), + scan_docker_image_unknown_low_medium_vulnerabilities_step(docker_image), scan_docker_image_high_critical_vulnerabilities_step(docker_image), slack_job_failed_step("grafana-backend-ops", docker_image), ], ) -def scan_docker_image_unkown_low_medium_vulnerabilities_step(docker_image): - return { - "name": "scan-unkown-low-medium-vulnerabilities", - "image": aquasec_trivy_image, - "commands": [ - "trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM " + docker_image, +def scan_build_test_publish_docker_image_pipeline(): + """Generates a cronjob pipeline for nightly scans of grafana Docker images. + + Returns: + Drone cronjob pipeline. + """ + + return cron_job_pipeline( + cronName = "nightly", + name = "scan-build-test-and-publish-docker-images", + steps = [ + scan_docker_image_unknown_low_medium_vulnerabilities_step("all"), + scan_docker_image_high_critical_vulnerabilities_step("all"), + slack_job_failed_step("grafana-backend-ops", "build-images"), ], + ) + +def scan_docker_image_unknown_low_medium_vulnerabilities_step(docker_image): + """Generates a step for scans of Grafana Docker images. + + Args: + docker_image: determines which image is scanned. + + Returns: + Drone cronjob step . + """ + + cmds = [] + if docker_image == "all": + for key in images: + cmds = cmds + ["trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM " + images[key]] + else: + cmds = ["trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM " + docker_image] + return { + "name": "scan-unknown-low-medium-vulnerabilities", + "image": aquasec_trivy_image, + "commands": cmds, } def scan_docker_image_high_critical_vulnerabilities_step(docker_image): + """Generates a step for scans of Grafana Docker images. + + Args: + docker_image: determines which image is scanned. + + Returns: + Drone cronjob step . + """ + + cmds = [] + if docker_image == "all": + for key in images: + cmds = cmds + ["trivy --exit-code 1 --severity HIGH,CRITICAL " + images[key]] + else: + cmds = ["trivy --exit-code 1 --severity HIGH,CRITICAL " + docker_image] return { "name": "scan-high-critical-vulnerabilities", "image": aquasec_trivy_image, - "commands": [ - "trivy --exit-code 1 --severity HIGH,CRITICAL " + docker_image, - ], + "commands": cmds, } def slack_job_failed_step(channel, image): return { "name": "slack-notify-failure", - "image": "plugins/slack", + "image": images["plugins_slack_image"], "settings": { "webhook": from_secret("slack_webhook_backend"), "channel": channel, @@ -95,7 +142,7 @@ def slack_job_failed_step(channel, image): def post_to_grafana_com_step(): return { "name": "post-to-grafana-com", - "image": publish_image, + "image": images["publish_image"], "environment": { "GRAFANA_COM_API_KEY": from_secret("grafana_api_key"), "GCP_KEY": from_secret("gcp_key"), diff --git a/scripts/drone/events/release.star b/scripts/drone/events/release.star index 1749e8312ff..aae0ed6428b 100644 --- a/scripts/drone/events/release.star +++ b/scripts/drone/events/release.star @@ -9,11 +9,9 @@ load( "build_docker_images_step", "build_frontend_package_step", "build_frontend_step", - "build_image", "build_plugins_step", "build_storybook_step", "clone_enterprise_step", - "cloudsdk_image", "compile_build_cmd", "copy_packages_for_docker_step", "download_grabpl_step", @@ -29,7 +27,6 @@ load( "package_step", "postgres_integration_tests_step", "publish_grafanacom_step", - "publish_image", "publish_images_step", "publish_linux_packages_step", "redis_integration_tests_step", @@ -63,6 +60,10 @@ load( "test_backend_enterprise", ) load("scripts/drone/vault.star", "from_secret", "prerelease_bucket") +load( + "scripts/drone/utils/images.star", + "images", +) ver_mode = "release" release_trigger = { @@ -84,7 +85,7 @@ release_trigger = { def store_npm_packages_step(): return { "name": "store-npm-packages", - "image": build_image, + "image": images["build_image"], "depends_on": [ "compile-build-cmd", "build-frontend-packages", @@ -99,7 +100,7 @@ def store_npm_packages_step(): def retrieve_npm_packages_step(): return { "name": "retrieve-npm-packages", - "image": publish_image, + "image": images["publish_image"], "depends_on": [ "compile-build-cmd", "yarn-install", @@ -115,7 +116,7 @@ def retrieve_npm_packages_step(): def release_npm_packages_step(): return { "name": "release-npm-packages", - "image": build_image, + "image": images["build_image"], "depends_on": [ "compile-build-cmd", "retrieve-npm-packages", @@ -554,7 +555,7 @@ def publish_artifacts_step(mode): security = "--security " return { "name": "publish-artifacts", - "image": publish_image, + "image": images["publish_image"], "environment": { "GCP_KEY": from_secret("gcp_key"), "PRERELEASE_BUCKET": from_secret("prerelease_bucket"), @@ -572,7 +573,7 @@ def publish_artifacts_step(mode): def publish_static_assets_step(): return { "name": "publish-static-assets", - "image": publish_image, + "image": images["publish_image"], "environment": { "GCP_KEY": from_secret("gcp_key"), "PRERELEASE_BUCKET": from_secret("prerelease_bucket"), @@ -587,7 +588,7 @@ def publish_static_assets_step(): def publish_storybook_step(): return { "name": "publish-storybook", - "image": publish_image, + "image": images["publish_image"], "environment": { "GCP_KEY": from_secret("gcp_key"), "PRERELEASE_BUCKET": from_secret("prerelease_bucket"), @@ -817,7 +818,7 @@ def verify_release_pipeline( step = { "name": "gsutil-stat", "depends_on": ["clone"], - "image": cloudsdk_image, + "image": images["cloudsdk_image"], "environment": { "BUCKET": bucket, "GCP_KEY": gcp_key, diff --git a/scripts/drone/pipelines/aws_marketplace.star b/scripts/drone/pipelines/aws_marketplace.star index a9ca6c57ab4..82d40a4fa57 100644 --- a/scripts/drone/pipelines/aws_marketplace.star +++ b/scripts/drone/pipelines/aws_marketplace.star @@ -5,18 +5,21 @@ This module contains steps and pipelines publishing to AWS Marketplace. load( "scripts/drone/steps/lib.star", "compile_build_cmd", - "publish_image", ) load("scripts/drone/vault.star", "from_secret") load( "scripts/drone/utils/utils.star", "pipeline", ) +load( + "scripts/drone/utils/images.star", + "images", +) def publish_aws_marketplace_step(): return { "name": "publish-aws-marketplace", - "image": publish_image, + "image": images["publish_image"], "commands": ["./bin/build publish aws --image grafana/grafana-enterprise --repo grafana-labs/grafanaenterprise --product 422b46fb-bea6-4f27-8bcc-832117bd627e"], "depends_on": ["compile-build-cmd"], "environment": { diff --git a/scripts/drone/pipelines/ci_images.star b/scripts/drone/pipelines/ci_images.star index fa6f458cd2b..734b731c2a8 100644 --- a/scripts/drone/pipelines/ci_images.star +++ b/scripts/drone/pipelines/ci_images.star @@ -2,10 +2,6 @@ This module contains steps and pipelines relating to creating CI Docker images. """ -load( - "scripts/drone/steps/lib.star", - "wix_image", -) load( "scripts/drone/utils/utils.star", "pipeline", @@ -14,6 +10,10 @@ load( "scripts/drone/vault.star", "from_secret", ) +load( + "scripts/drone/utils/images.star", + "images", +) def publish_ci_windows_test_image_pipeline(): trigger = { @@ -28,7 +28,7 @@ def publish_ci_windows_test_image_pipeline(): steps = [ { "name": "clone", - "image": wix_image, + "image": images["wix_image"], "environment": { "GITHUB_TOKEN": from_secret("github_token"), }, @@ -39,7 +39,7 @@ def publish_ci_windows_test_image_pipeline(): }, { "name": "build-and-publish", - "image": "docker:windowsservercore-1809", + "image": images["windows_server_core_image"], "environment": { "DOCKER_USERNAME": from_secret("docker_username"), "DOCKER_PASSWORD": from_secret("docker_password"), diff --git a/scripts/drone/pipelines/docs.star b/scripts/drone/pipelines/docs.star index 5113fc09de2..d60fcdb6cad 100644 --- a/scripts/drone/pipelines/docs.star +++ b/scripts/drone/pipelines/docs.star @@ -5,11 +5,14 @@ This module returns all the pipelines used in the event of documentation changes load( "scripts/drone/steps/lib.star", "build_docs_website_step", - "build_image", "codespell_step", "identify_runner_step", "yarn_install_step", ) +load( + "scripts/drone/utils/images.star", + "images", +) load( "scripts/drone/utils/utils.star", "pipeline", @@ -46,7 +49,7 @@ def docs_pipelines(ver_mode, trigger): def lint_docs(): return { "name": "lint-docs", - "image": build_image, + "image": images["build_image"], "depends_on": [ "yarn-install", ], diff --git a/scripts/drone/pipelines/github.star b/scripts/drone/pipelines/github.star index d3ccb6b30c9..fc43638c504 100644 --- a/scripts/drone/pipelines/github.star +++ b/scripts/drone/pipelines/github.star @@ -6,18 +6,21 @@ load( "scripts/drone/steps/lib.star", "compile_build_cmd", "fetch_images_step", - "publish_image", ) load("scripts/drone/vault.star", "from_secret") load( "scripts/drone/utils/utils.star", "pipeline", ) +load( + "scripts/drone/utils/images.star", + "images", +) def publish_github_step(): return { "name": "publish-github", - "image": publish_image, + "image": images["publish_image"], "commands": ["./bin/build publish github --repo $${GH_REGISTRY} --create"], "depends_on": ["fetch-images-enterprise2"], "environment": { diff --git a/scripts/drone/pipelines/shellcheck.star b/scripts/drone/pipelines/shellcheck.star index 2a2aa55149e..b8ef57265d7 100644 --- a/scripts/drone/pipelines/shellcheck.star +++ b/scripts/drone/pipelines/shellcheck.star @@ -2,11 +2,15 @@ This module returns a Drone step and pipeline for linting with shellcheck. """ -load("scripts/drone/steps/lib.star", "build_image", "compile_build_cmd") +load("scripts/drone/steps/lib.star", "compile_build_cmd") load( "scripts/drone/utils/utils.star", "pipeline", ) +load( + "scripts/drone/utils/images.star", + "images", +) trigger = { "event": [ @@ -25,7 +29,7 @@ trigger = { def shellcheck_step(): return { "name": "shellcheck", - "image": build_image, + "image": images["build_image"], "depends_on": [ "compile-build-cmd", ], diff --git a/scripts/drone/pipelines/windows.star b/scripts/drone/pipelines/windows.star index f00b0a16fd1..e2ecd7a4f22 100644 --- a/scripts/drone/pipelines/windows.star +++ b/scripts/drone/pipelines/windows.star @@ -10,11 +10,14 @@ load( "scripts/drone/steps/lib.star", "get_windows_steps", "windows_clone_step", - "windows_go_image", "windows_init_enterprise_steps", "windows_test_backend_step", "windows_wire_install_step", ) +load( + "scripts/drone/utils/images.star", + "images", +) def windows_test_backend(trigger, edition, ver_mode): """ Generates a pipeline that runs backend tests on Windows @@ -36,7 +39,7 @@ def windows_test_backend(trigger, edition, ver_mode): else: steps.extend([{ "name": "windows-init", - "image": windows_go_image, + "image": images["windows_go_image"], "depends_on": ["clone"], "commands": [], }]) diff --git a/scripts/drone/services/services.star b/scripts/drone/services/services.star index 93113810157..0140ad0b433 100644 --- a/scripts/drone/services/services.star +++ b/scripts/drone/services/services.star @@ -2,6 +2,11 @@ This module has functions for Drone services to be used in pipelines. """ +load( + "scripts/drone/utils/images.star", + "images", +) + def integration_test_services_volumes(): return [ {"name": "postgres", "temp": {"medium": "memory"}}, @@ -13,7 +18,7 @@ def integration_test_services(): services = [ { "name": "postgres", - "image": "postgres:12.3-alpine", + "image": images["postgres_alpine_image"], "environment": { "POSTGRES_USER": "grafanatest", "POSTGRES_PASSWORD": "grafanatest", @@ -26,7 +31,7 @@ def integration_test_services(): }, { "name": "mysql57", - "image": "mysql:5.7.39", + "image": images["mysql5_image"], "environment": { "MYSQL_ROOT_PASSWORD": "rootpass", "MYSQL_DATABASE": "grafana_tests", @@ -37,7 +42,7 @@ def integration_test_services(): }, { "name": "mysql80", - "image": "mysql:8.0.32", + "image": images["mysql8_image"], "environment": { "MYSQL_ROOT_PASSWORD": "rootpass", "MYSQL_DATABASE": "grafana_tests", @@ -49,12 +54,12 @@ def integration_test_services(): }, { "name": "redis", - "image": "redis:6.2.11-alpine", + "image": images["redis_alpine_image"], "environment": {}, }, { "name": "memcached", - "image": "memcached:1.6.9-alpine", + "image": images["memcached_alpine_image"], "environment": {}, }, ] @@ -64,7 +69,7 @@ def integration_test_services(): def ldap_service(): return { "name": "ldap", - "image": "osixia/openldap:1.4.0", + "image": images["openldap_image"], "environment": { "LDAP_ADMIN_PASSWORD": "grafana", "LDAP_DOMAIN": "grafana.org", diff --git a/scripts/drone/steps/lib.star b/scripts/drone/steps/lib.star index ba6a38cbdb5..3962936c21e 100644 --- a/scripts/drone/steps/lib.star +++ b/scripts/drone/steps/lib.star @@ -7,18 +7,12 @@ load( "from_secret", "prerelease_bucket", ) +load( + "scripts/drone/utils/images.star", + "images", +) grabpl_version = "v3.0.38" -cloudsdk_image = "google/cloud-sdk:431.0.0" -build_image = "grafana/build-container:1.7.4" -publish_image = "grafana/grafana-ci-deploy:1.3.3" -deploy_docker_image = "us.gcr.io/kubernetes-dev/drone/plugins/deploy-image" -alpine_image = "alpine:3.17.1" -curl_image = "byrnedo/alpine-curl:0.1.8" -windows_image = "mcr.microsoft.com/windows:1809" -wix_image = "grafana/ci-wix:0.1.1" -go_image = "golang:1.20.4" -windows_go_image = "grafana/grafana-ci-windows-test:0.1.0" trigger_oss = { "repo": [ @@ -29,7 +23,7 @@ trigger_oss = { def slack_step(channel, template, secret): return { "name": "slack", - "image": "plugins/slack", + "image": images["plugins_slack_image"], "settings": { "webhook": from_secret(secret), "channel": channel, @@ -40,7 +34,7 @@ def slack_step(channel, template, secret): def yarn_install_step(): return { "name": "yarn-install", - "image": build_image, + "image": images["build_image"], "commands": [ "yarn install --immutable", ], @@ -50,7 +44,7 @@ def yarn_install_step(): def wire_install_step(): return { "name": "wire-install", - "image": build_image, + "image": images["build_image"], "commands": [ "make gen-go", ], @@ -62,7 +56,7 @@ def wire_install_step(): def windows_wire_install_step(edition): return { "name": "wire-install", - "image": windows_go_image, + "image": images["windows_go_image"], "commands": [ "go install github.com/google/wire/cmd/wire@v0.5.0", "wire gen -tags {} ./pkg/server".format(edition), @@ -76,7 +70,7 @@ def identify_runner_step(platform = "linux"): if platform == "linux": return { "name": "identify-runner", - "image": alpine_image, + "image": images["alpine_image"], "commands": [ "echo $DRONE_RUNNER_NAME", ], @@ -84,7 +78,7 @@ def identify_runner_step(platform = "linux"): else: return { "name": "identify-runner", - "image": windows_image, + "image": images["windows_image"], "commands": [ "echo $env:DRONE_RUNNER_NAME", ], @@ -111,7 +105,7 @@ def clone_enterprise_step(source = "${DRONE_COMMIT}"): """ step = { "name": "clone-enterprise", - "image": build_image, + "image": images["build_image"], "environment": { "GITHUB_TOKEN": from_secret("github_token"), }, @@ -137,7 +131,7 @@ def clone_enterprise_step_pr(source = "${DRONE_COMMIT}", target = "main", canFai """ step = { "name": "clone-enterprise", - "image": build_image, + "image": images["build_image"], "environment": { "GITHUB_TOKEN": from_secret("github_token"), }, @@ -183,7 +177,7 @@ def init_enterprise_step(ver_mode): token = "" return { "name": "init-enterprise", - "image": build_image, + "image": images["build_image"], "depends_on": [ "clone-enterprise", "grabpl", @@ -241,7 +235,7 @@ def windows_init_enterprise_steps(ver_mode): download_grabpl_step(platform = "windows"), { "name": "clone", - "image": wix_image, + "image": images["wix_image"], "environment": { "GITHUB_TOKEN": from_secret("github_token"), }, @@ -249,7 +243,7 @@ def windows_init_enterprise_steps(ver_mode): }, { "name": "windows-init", - "image": wix_image, + "image": images["wix_image"], "commands": init_cmds, "depends_on": ["clone"], "environment": {"GITHUB_TOKEN": from_secret("github_token")}, @@ -262,7 +256,7 @@ def download_grabpl_step(platform = "linux"): if platform == "windows": return { "name": "grabpl", - "image": wix_image, + "image": images["wix_image"], "commands": [ '$$ProgressPreference = "SilentlyContinue"', "Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/{}/windows/grabpl.exe -OutFile grabpl.exe".format( @@ -273,7 +267,7 @@ def download_grabpl_step(platform = "linux"): return { "name": "grabpl", - "image": curl_image, + "image": images["curl_image"], "commands": [ "mkdir -p bin", "curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/{}/grabpl".format( @@ -286,7 +280,7 @@ def download_grabpl_step(platform = "linux"): def lint_drone_step(): return { "name": "lint-drone", - "image": curl_image, + "image": images["curl_image"], "commands": [ "./bin/build verify-drone", ], @@ -298,7 +292,7 @@ def lint_drone_step(): def lint_starlark_step(): return { "name": "lint-starlark", - "image": build_image, + "image": images["build_image"], "commands": [ "./bin/build verify-starlark .", ], @@ -325,7 +319,7 @@ def enterprise_downstream_step(ver_mode): step = { "name": "trigger-enterprise-downstream", - "image": "grafana/drone-downstream", + "image": images["drone_downstream_image"], "settings": { "server": "https://drone.grafana.net", "token": from_secret("drone_token"), @@ -349,7 +343,7 @@ def lint_backend_step(): return { "name": "lint-backend", # TODO: build_image or go_image? - "image": go_image, + "image": images["go_image"], "environment": { # We need CGO because of go-sqlite3 "CGO_ENABLED": "1", @@ -367,7 +361,7 @@ def lint_backend_step(): def validate_modfile_step(): return { "name": "validate-modfile", - "image": go_image, + "image": images["go_image"], "failure": "ignore", "commands": [ "go run scripts/modowners/modowners.go check go.mod", @@ -377,7 +371,7 @@ def validate_modfile_step(): def benchmark_ldap_step(): return { "name": "benchmark-ldap", - "image": build_image, + "image": images["build_image"], "environment": { "LDAP_HOSTNAME": "ldap", }, @@ -390,7 +384,7 @@ def benchmark_ldap_step(): def build_storybook_step(ver_mode): return { "name": "build-storybook", - "image": build_image, + "image": images["build_image"], "depends_on": [ # Best to ensure that this step doesn't mess with what's getting built and packaged "build-frontend", @@ -434,7 +428,7 @@ def store_storybook_step(ver_mode, trigger = None): step = { "name": "store-storybook", - "image": publish_image, + "image": images["publish_image"], "depends_on": [ "build-storybook", ] + @@ -464,7 +458,7 @@ def store_storybook_step(ver_mode, trigger = None): def e2e_tests_artifacts(): return { "name": "e2e-tests-artifacts-upload", - "image": cloudsdk_image, + "image": images["cloudsdk_image"], "depends_on": [ "end-to-end-tests-dashboards-suite", "end-to-end-tests-panels-suite", @@ -526,7 +520,7 @@ def upload_cdn_step(edition, ver_mode, trigger = None): step = { "name": "upload-cdn-assets" + enterprise2_suffix(edition), - "image": publish_image, + "image": images["publish_image"], "depends_on": deps, "environment": { "GCP_KEY": from_secret("gcp_key"), @@ -578,7 +572,7 @@ def build_backend_step(edition, ver_mode, variants = None): return { "name": "build-backend" + enterprise2_suffix(edition), - "image": build_image, + "image": images["build_image"], "depends_on": [ "wire-install", "compile-build-cmd", @@ -613,7 +607,7 @@ def build_frontend_step(edition, ver_mode): return { "name": "build-frontend", - "image": build_image, + "image": images["build_image"], "environment": { "NODE_OPTIONS": "--max_old_space_size=8192", }, @@ -653,7 +647,7 @@ def build_frontend_package_step(edition, ver_mode): return { "name": "build-frontend-packages", - "image": build_image, + "image": images["build_image"], "environment": { "NODE_OPTIONS": "--max_old_space_size=8192", }, @@ -673,7 +667,7 @@ def build_plugins_step(edition, ver_mode): env = None return { "name": "build-plugins", - "image": build_image, + "image": images["build_image"], "environment": env, "depends_on": [ "compile-build-cmd", @@ -685,7 +679,7 @@ def build_plugins_step(edition, ver_mode): ], } -def test_backend_step(image = build_image): +def test_backend_step(image = images["build_image"]): return { "name": "test-backend", "image": image, @@ -698,13 +692,13 @@ def test_backend_step(image = build_image): } def windows_test_backend_step(): - step = test_backend_step(image = windows_go_image) + step = test_backend_step(image = images["windows_go_image"]) return step def test_backend_integration_step(): return { "name": "test-backend-integration", - "image": build_image, + "image": images["build_image"], "depends_on": [ "wire-install", ], @@ -729,7 +723,7 @@ def betterer_frontend_step(edition = "oss"): deps.extend(["yarn-install"]) return { "name": "betterer-frontend", - "image": build_image, + "image": images["build_image"], "depends_on": deps, "commands": [ "yarn betterer ci", @@ -752,7 +746,7 @@ def test_frontend_step(edition = "oss"): deps.extend(["yarn-install"]) return { "name": "test-frontend", - "image": build_image, + "image": images["build_image"], "environment": { "TEST_MAX_WORKERS": "50%", }, @@ -765,7 +759,7 @@ def test_frontend_step(edition = "oss"): def lint_frontend_step(): return { "name": "lint-frontend", - "image": build_image, + "image": images["build_image"], "environment": { "TEST_MAX_WORKERS": "50%", }, @@ -784,7 +778,7 @@ def verify_i18n_step(): uncommited_error_message = "\nTranslation extraction has not been committed. Please run 'yarn i18n:extract', commit the changes and push again." return { "name": "verify-i18n", - "image": build_image, + "image": images["build_image"], "depends_on": [ "yarn-install", ], @@ -836,7 +830,7 @@ def test_a11y_frontend_step(ver_mode, port = 3001): return { "name": "test-a11y-frontend", # TODO which image should be used? - "image": "grafana/docker-puppeteer:1.1.0", + "image": images["docker_puppeteer_image"], "depends_on": [ "grafana-server", ], @@ -861,7 +855,7 @@ def frontend_metrics_step(trigger = None): """ step = { "name": "publish-frontend-metrics", - "image": build_image, + "image": images["build_image"], "depends_on": [ "test-a11y-frontend", ], @@ -880,7 +874,7 @@ def frontend_metrics_step(trigger = None): def codespell_step(): return { "name": "codespell", - "image": build_image, + "image": images["build_image"], "commands": [ # Important: all words have to be in lowercase, and separated by "\n". 'echo -e "unknwon\nreferer\nerrorstring\neror\niam\nwan" > words_to_ignore.txt', @@ -939,7 +933,7 @@ def package_step(edition, ver_mode): return { "name": "package" + enterprise2_suffix(edition), - "image": build_image, + "image": images["build_image"], "depends_on": deps, "environment": env, "commands": cmds, @@ -962,7 +956,7 @@ def grafana_server_step(edition, port = 3001): return { "name": "grafana-server", - "image": build_image, + "image": images["build_image"], "detach": True, "depends_on": [ "build-plugins", @@ -982,7 +976,7 @@ def e2e_tests_step(suite, port = 3001, tries = None): cmd += " --tries {}".format(tries) return { "name": "end-to-end-tests-{}".format(suite), - "image": "cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97", + "image": images["cypress_image"], "depends_on": [ "grafana-server", ], @@ -1034,7 +1028,7 @@ def cloud_plugins_e2e_tests_step(suite, cloud, trigger = None): branch = "${DRONE_SOURCE_BRANCH}".replace("/", "-") step = { "name": "end-to-end-tests-{}-{}".format(suite, cloud), - "image": "us-docker.pkg.dev/grafanalabs-dev/cloud-data-sources/e2e:latest", + "image": images["cloud_datasources_e2e_image"], "depends_on": [ "grafana-server", ], @@ -1048,7 +1042,7 @@ def build_docs_website_step(): return { "name": "build-docs-website", # Use latest revision here, since we want to catch if it breaks - "image": "grafana/docs-base:latest", + "image": images["docs_image"], "commands": [ "mkdir -p /hugo/content/docs/grafana/latest", "cp -r docs/sources/* /hugo/content/docs/grafana/latest/", @@ -1059,7 +1053,7 @@ def build_docs_website_step(): def copy_packages_for_docker_step(edition = None): return { "name": "copy-packages-for-docker", - "image": build_image, + "image": images["build_image"], "depends_on": [ "package" + enterprise2_suffix(edition), ], @@ -1107,7 +1101,7 @@ def build_docker_images_step(edition, archs = None, ubuntu = False, publish = Fa return { "name": "build-docker-images" + ubuntu_sfx, - "image": cloudsdk_image, + "image": images["cloudsdk_image"], "depends_on": [ "copy-packages-for-docker", "compile-build-cmd", @@ -1120,7 +1114,7 @@ def build_docker_images_step(edition, archs = None, ubuntu = False, publish = Fa def fetch_images_step(edition): return { "name": "fetch-images-{}".format(edition), - "image": cloudsdk_image, + "image": images["cloudsdk_image"], "environment": { "GCP_KEY": from_secret("gcp_key"), "DOCKER_USER": from_secret("docker_username"), @@ -1199,7 +1193,7 @@ def publish_images_step(edition, ver_mode, mode, docker_repo, trigger = None): step = { "name": "publish-images-{}".format(name), - "image": cloudsdk_image, + "image": images["cloudsdk_image"], "environment": environment, "commands": [cmd], "depends_on": deps, @@ -1224,7 +1218,7 @@ def postgres_integration_tests_step(): ] return { "name": "postgres-integration-tests", - "image": build_image, + "image": images["build_image"], "depends_on": ["wire-install"], "environment": { "PGPASSWORD": "grafanatest", @@ -1245,7 +1239,7 @@ def mysql_integration_tests_step(hostname, version): ] return { "name": "mysql-{}-integration-tests".format(version), - "image": build_image, + "image": images["build_image"], "depends_on": ["wire-install"], "environment": { "GRAFANA_TEST_DB": "mysql", @@ -1257,7 +1251,7 @@ def mysql_integration_tests_step(hostname, version): def redis_integration_tests_step(): return { "name": "redis-integration-tests", - "image": build_image, + "image": images["build_image"], "depends_on": ["wire-install"], "environment": { "REDIS_URL": "redis://redis:6379/0", @@ -1272,7 +1266,7 @@ def redis_integration_tests_step(): def memcached_integration_tests_step(): return { "name": "memcached-integration-tests", - "image": build_image, + "image": images["build_image"], "depends_on": ["wire-install"], "environment": { "MEMCACHED_HOSTS": "memcached:11211", @@ -1296,7 +1290,7 @@ def release_canary_npm_packages_step(trigger = None): """ step = { "name": "release-canary-npm-packages", - "image": build_image, + "image": images["build_image"], "depends_on": end_to_end_tests_deps(), "environment": { "NPM_TOKEN": from_secret("npm_token"), @@ -1339,7 +1333,7 @@ def upload_packages_step(edition, ver_mode, trigger = None): """ step = { "name": "upload-packages" + enterprise2_suffix(edition), - "image": publish_image, + "image": images["publish_image"], "depends_on": end_to_end_tests_deps(), "environment": { "GCP_KEY": from_secret("gcp_key"), @@ -1381,7 +1375,7 @@ def publish_grafanacom_step(edition, ver_mode): return { "name": "publish-grafanacom-{}".format(edition), - "image": publish_image, + "image": images["publish_image"], "depends_on": [ "publish-linux-packages-deb", "publish-linux-packages-rpm", @@ -1399,7 +1393,7 @@ def publish_linux_packages_step(edition, package_manager = "deb"): return { "name": "publish-linux-packages-{}".format(package_manager), # See https://github.com/grafana/deployment_tools/blob/master/docker/package-publish/README.md for docs on that image - "image": "us.gcr.io/kubernetes-dev/package-publish:latest", + "image": images["package_publish_image"], "depends_on": ["compile-build-cmd"], "privileged": True, "settings": { @@ -1421,7 +1415,7 @@ def publish_linux_packages_step(edition, package_manager = "deb"): def windows_clone_step(): return { "name": "clone", - "image": wix_image, + "image": images["wix_image"], "environment": { "GITHUB_TOKEN": from_secret("github_token"), }, @@ -1481,7 +1475,7 @@ def get_windows_steps(edition, ver_mode): [ { "name": "clone", - "image": wix_image, + "image": images["wix_image"], "environment": { "GITHUB_TOKEN": from_secret("github_token"), }, @@ -1489,7 +1483,7 @@ def get_windows_steps(edition, ver_mode): }, { "name": "windows-init", - "image": wix_image, + "image": images["wix_image"], "commands": init_cmds, "depends_on": ["clone"], "environment": {"GITHUB_TOKEN": from_secret("github_token")}, @@ -1508,7 +1502,7 @@ def get_windows_steps(edition, ver_mode): [ { "name": "windows-init", - "image": wix_image, + "image": images["wix_image"], "commands": init_cmds, }, ], @@ -1583,7 +1577,7 @@ def get_windows_steps(edition, ver_mode): steps.append( { "name": "build-windows-installer", - "image": wix_image, + "image": images["wix_image"], "depends_on": [ "windows-init", ], @@ -1601,7 +1595,7 @@ def get_windows_steps(edition, ver_mode): def verify_gen_cue_step(): return { "name": "verify-gen-cue", - "image": build_image, + "image": images["build_image"], "depends_on": [], "commands": [ "# It is required that code generated from Thema/CUE be committed and in sync with its inputs.", @@ -1613,7 +1607,7 @@ def verify_gen_cue_step(): def verify_gen_jsonnet_step(): return { "name": "verify-gen-jsonnet", - "image": build_image, + "image": images["build_image"], "depends_on": [], "commands": [ "# It is required that generated jsonnet is committed and in sync with its inputs.", @@ -1625,7 +1619,7 @@ def verify_gen_jsonnet_step(): def trigger_test_release(): return { "name": "trigger-test-release", - "image": build_image, + "image": images["build_image"], "environment": { "GITHUB_TOKEN": from_secret("github_token_pr"), "TEST_TAG": "v0.0.0-test", @@ -1659,7 +1653,7 @@ def trigger_test_release(): def artifacts_page_step(): return { "name": "artifacts-page", - "image": build_image, + "image": images["build_image"], "depends_on": [ "compile-build-cmd", ], @@ -1687,7 +1681,7 @@ def compile_build_cmd(edition = "oss"): ] return { "name": "compile-build-cmd", - "image": go_image, + "image": images["go_image"], "commands": [ "go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd", ], diff --git a/scripts/drone/utils/images.star b/scripts/drone/utils/images.star new file mode 100644 index 00000000000..0450301c1c2 --- /dev/null +++ b/scripts/drone/utils/images.star @@ -0,0 +1,29 @@ +""" +This module contains all the docker images that are used to build test and publish Grafana. +""" + +images = { + "cloudsdk_image": "google/cloud-sdk:431.0.0", + "build_image": "grafana/build-container:1.7.4", + "publish_image": "grafana/grafana-ci-deploy:1.3.3", + "alpine_image": "alpine:3.17.1", + "curl_image": "byrnedo/alpine-curl:0.1.8", + "windows_image": "mcr.microsoft.com/windows:1809", + "wix_image": "grafana/ci-wix:0.1.1", + "go_image": "golang:1.20.4", + "windows_go_image": "grafana/grafana-ci-windows-test:0.1.0", + "plugins_slack_image": "plugins/slack", + "postgres_alpine_image": "postgres:12.3-alpine", + "mysql5_image": "mysql:5.7.39", + "mysql8_image": "mysql:8.0.32", + "redis_alpine_image": "redis:6.2.11-alpine", + "memcached_alpine_image": "memcached:1.6.9-alpine", + "windows_server_core_image": "docker:windowsservercore-1809", + "package_publish_image": "us.gcr.io/kubernetes-dev/package-publish:latest", + "openldap_image": "osixia/openldap:1.4.0", + "drone_downstream_image": "grafana/drone-downstream", + "docker_puppeteer_image": "grafana/docker-puppeteer:1.1.0", + "docs_image": "grafana/docs-base:latest", + "cypress_image": "cypress/included:9.5.1-node16.14.0-slim-chrome99-ff97", + "cloud_datasources_e2e_image": "us-docker.pkg.dev/grafanalabs-dev/cloud-data-sources/e2e:latest", +}