Automation: Verify DEB and RPM packages (#90146)

* baldm0mma/verify_aptyum/ add verify_linux_packages_step

* baldm0mma/verify_aptyum/ add name and image

* baldm0mma/verify_aptyum/ add commands

* baldm0mma/verify_aptyum/ add test pipeline

* baldm0mma/verify_aptyum/ update deps for verify_linux_packages_step

* baldm0mma/verify_aptyum/ update ubuntu image

* baldm0mma/verify_aptyum/ add rockylinux9 image

* baldm0mma/verify_aptyum/ update to verify_linux_DEB_packages_step naming

* baldm0mma/verify_aptyum/ add verify_linux_RPM_packages_step

* baldm0mma/verify_aptyum/ update commands for RPM check

* baldm0mma/verify_aptyum/ update test pipeline

* baldm0mma/verify_aptyum/ update annos

* baldm0mma/verify_aptyum/ update annos

* baldm0mma/verify_aptyum/ update args

* baldm0mma/verify_aptyum/ add arg to verify_linux_RPM_packages_step

* baldm0mma/verify_aptyum/ update oss_steps

* baldm0mma/verify_aptyum/ update deps args

* baldm0mma/verify_aptyum/ update sec hash

* baldm0mma/verify_aptyum/ make format-drone

* baldm0mma/verify_aptyum/ update sec hash

* baldm0mma/verify_aptyum/ update drone

* baldm0mma/verify_aptyum/ update function styles

* baldm0mma/verify_aptyum/ revert changes

* baldm0mma/verify_aptyum/ make drone

* baldm0mma/verify_aptyum/ redirect install logs

* baldm0mma/verify_aptyum/ restructure args

* baldm0mma/verify_aptyum/ update verify_linux_DEB_packages_step

* baldm0mma/verify_aptyum/ make format drone

* baldm0mma/verify_aptyum/ update verify_linux_RPM_packages_step

* baldm0mma/verify_aptyum/ add retry_command

* baldm0mma/verify_aptyum/ make drone

* baldm0mma/verify_aptyum/ update attempts to 10 minutes

* baldm0mma/verify_aptyum/ handle is_preview

* baldm0mma/verify_aptyum/ add doc strings to satisfy starlark linter

* baldm0mma/verify_aptyum/ revert is_preview

* baldm0mma/verify_aptyum/ update to rpm repo download

* baldm0mma/verify_aptyum/ add back logging for rpm

* baldm0mma/verify_aptyum/ revert to working

* baldm0mma/verify_aptyum/ update to use RPM repo

* baldm0mma/verify_aptyum/ remove redirect for logging

* baldm0mma/verify_aptyum/ rem gpg check

* baldm0mma/verify_aptyum/ fromat drone

* baldm0mma/verify_aptyum/ rem comment

* baldm0mma/verify_aptyum/ add single comment on timing

.drone.yml

@@ -3139,6 +3139,130 @@ volumes:
     path: /var/run/docker.sock
   name: docker
 ---
+clone:
+  retries: 3
+depends_on: []
+image_pull_secrets:
+- gcr
+- gar
+kind: pipeline
+name: verify-linux-packages
+node:
+  type: no-parallel
+platform:
+  arch: amd64
+  os: linux
+services: []
+steps:
+- commands:
+  - 'echo "Step 1: Updating package lists..."'
+  - apt-get update >/dev/null 2>&1
+  - 'echo "Step 2: Installing prerequisites..."'
+  - DEBIAN_FRONTEND=noninteractive apt-get install -yq apt-transport-https software-properties-common
+    wget >/dev/null 2>&1
+  - 'echo "Step 3: Adding Grafana GPG key..."'
+  - mkdir -p /etc/apt/keyrings/
+  - wget -q -O - https://apt.grafana.com/gpg.key | gpg --dearmor | tee /etc/apt/keyrings/grafana.gpg
+    > /dev/null
+  - 'echo "Step 4: Adding Grafana repository..."'
+  - echo "deb [signed-by=/etc/apt/keyrings/grafana.gpg] https://apt.grafana.com stable
+    main" | tee -a /etc/apt/sources.list.d/grafana.list
+  - 'echo "Step 5: Installing Grafana..."'
+  - for i in $(seq 1 10); do
+  - '    if apt-get update >/dev/null 2>&1 && DEBIAN_FRONTEND=noninteractive apt-get
+    install -yq grafana=${TAG} >/dev/null 2>&1; then'
+  - '        echo "Command succeeded on attempt $i"'
+  - '        break'
+  - '    else'
+  - '        echo "Attempt $i failed"'
+  - '        if [ $i -eq 10 ]; then'
+  - '            echo ''All attempts failed'''
+  - '            exit 1'
+  - '        fi'
+  - '        echo "Waiting 60 seconds before next attempt..."'
+  - '        sleep 60'
+  - '    fi'
+  - done
+  - 'echo "Step 6: Verifying Grafana installation..."'
+  - 'if dpkg -s grafana | grep -q "Version: ${TAG}"; then'
+  - '    echo "Successfully verified Grafana version ${TAG}"'
+  - else
+  - '    echo "Failed to verify Grafana version ${TAG}"'
+  - '    exit 1'
+  - fi
+  - echo "Verification complete."
+  depends_on: []
+  environment: {}
+  image: ubuntu:22.04
+  name: verify-linux-DEB-packages
+- commands:
+  - 'echo "Step 1: Updating package lists..."'
+  - dnf check-update -y >/dev/null 2>&1 || true
+  - 'echo "Step 2: Installing prerequisites..."'
+  - dnf install -y dnf-utils >/dev/null 2>&1
+  - 'echo "Step 3: Adding Grafana GPG key..."'
+  - rpm --import https://rpm.grafana.com/gpg.key
+  - 'echo "Step 4: Configuring Grafana repository..."'
+  - |-
+    echo '[grafana]
+    name=grafana
+    baseurl=https://rpm.grafana.com
+    repo_gpgcheck=0
+    enabled=1
+    gpgcheck=0
+    gpgkey=https://rpm.grafana.com/gpg.key
+    sslverify=1
+    sslcacert=/etc/pki/tls/certs/ca-bundle.crt
+    ' > /etc/yum.repos.d/grafana.repo
+  - 'echo "Step 5: Checking RPM repository..."'
+  - dnf list available grafana-${TAG}
+  - if [ $? -eq 0 ]; then
+  - '    echo "Grafana package found in repository. Installing from repo..."'
+  - for i in $(seq 1 5); do
+  - '    if dnf install -y --nogpgcheck grafana-${TAG} >/dev/null 2>&1; then'
+  - '        echo "Command succeeded on attempt $i"'
+  - '        break'
+  - '    else'
+  - '        echo "Attempt $i failed"'
+  - '        if [ $i -eq 5 ]; then'
+  - '            echo ''All attempts failed'''
+  - '            exit 1'
+  - '        fi'
+  - '        echo "Waiting 60 seconds before next attempt..."'
+  - '        sleep 60'
+  - '    fi'
+  - done
+  - '    echo "Verifying GPG key..."'
+  - '    rpm --import https://rpm.grafana.com/gpg.key'
+  - '    rpm -qa gpg-pubkey* | xargs rpm -qi | grep -i grafana'
+  - else
+  - '    echo "Grafana package version ${TAG} not found in repository."'
+  - '    dnf repolist'
+  - '    dnf list available grafana*'
+  - '    exit 1'
+  - fi
+  - 'echo "Step 6: Verifying Grafana installation..."'
+  - if rpm -q grafana | grep -q "${TAG}"; then
+  - '    echo "Successfully verified Grafana version ${TAG}"'
+  - else
+  - '    echo "Failed to verify Grafana version ${TAG}"'
+  - '    exit 1'
+  - fi
+  - echo "Verification complete."
+  depends_on: []
+  environment: {}
+  image: rockylinux:9
+  name: verify-linux-RPM-packages
+trigger:
+  event:
+  - promote
+  target: verify-linux-packages
+type: docker
+volumes:
+- host:
+    path: /var/run/docker.sock
+  name: docker
+---
 clone:
   retries: 3
 depends_on:
@@ -3207,6 +3331,107 @@ steps:
     service_account_json:
       from_secret: packages_service_account
     target_bucket: grafana-packages
+- commands:
+  - 'echo "Step 1: Updating package lists..."'
+  - apt-get update >/dev/null 2>&1
+  - 'echo "Step 2: Installing prerequisites..."'
+  - DEBIAN_FRONTEND=noninteractive apt-get install -yq apt-transport-https software-properties-common
+    wget >/dev/null 2>&1
+  - 'echo "Step 3: Adding Grafana GPG key..."'
+  - mkdir -p /etc/apt/keyrings/
+  - wget -q -O - https://apt.grafana.com/gpg.key | gpg --dearmor | tee /etc/apt/keyrings/grafana.gpg
+    > /dev/null
+  - 'echo "Step 4: Adding Grafana repository..."'
+  - echo "deb [signed-by=/etc/apt/keyrings/grafana.gpg] https://apt.grafana.com stable
+    main" | tee -a /etc/apt/sources.list.d/grafana.list
+  - 'echo "Step 5: Installing Grafana..."'
+  - for i in $(seq 1 10); do
+  - '    if apt-get update >/dev/null 2>&1 && DEBIAN_FRONTEND=noninteractive apt-get
+    install -yq grafana=${TAG} >/dev/null 2>&1; then'
+  - '        echo "Command succeeded on attempt $i"'
+  - '        break'
+  - '    else'
+  - '        echo "Attempt $i failed"'
+  - '        if [ $i -eq 10 ]; then'
+  - '            echo ''All attempts failed'''
+  - '            exit 1'
+  - '        fi'
+  - '        echo "Waiting 60 seconds before next attempt..."'
+  - '        sleep 60'
+  - '    fi'
+  - done
+  - 'echo "Step 6: Verifying Grafana installation..."'
+  - 'if dpkg -s grafana | grep -q "Version: ${TAG}"; then'
+  - '    echo "Successfully verified Grafana version ${TAG}"'
+  - else
+  - '    echo "Failed to verify Grafana version ${TAG}"'
+  - '    exit 1'
+  - fi
+  - echo "Verification complete."
+  depends_on:
+  - publish-linux-packages-deb
+  environment: {}
+  image: ubuntu:22.04
+  name: verify-linux-DEB-packages
+- commands:
+  - 'echo "Step 1: Updating package lists..."'
+  - dnf check-update -y >/dev/null 2>&1 || true
+  - 'echo "Step 2: Installing prerequisites..."'
+  - dnf install -y dnf-utils >/dev/null 2>&1
+  - 'echo "Step 3: Adding Grafana GPG key..."'
+  - rpm --import https://rpm.grafana.com/gpg.key
+  - 'echo "Step 4: Configuring Grafana repository..."'
+  - |-
+    echo '[grafana]
+    name=grafana
+    baseurl=https://rpm.grafana.com
+    repo_gpgcheck=0
+    enabled=1
+    gpgcheck=0
+    gpgkey=https://rpm.grafana.com/gpg.key
+    sslverify=1
+    sslcacert=/etc/pki/tls/certs/ca-bundle.crt
+    ' > /etc/yum.repos.d/grafana.repo
+  - 'echo "Step 5: Checking RPM repository..."'
+  - dnf list available grafana-${TAG}
+  - if [ $? -eq 0 ]; then
+  - '    echo "Grafana package found in repository. Installing from repo..."'
+  - for i in $(seq 1 5); do
+  - '    if dnf install -y --nogpgcheck grafana-${TAG} >/dev/null 2>&1; then'
+  - '        echo "Command succeeded on attempt $i"'
+  - '        break'
+  - '    else'
+  - '        echo "Attempt $i failed"'
+  - '        if [ $i -eq 5 ]; then'
+  - '            echo ''All attempts failed'''
+  - '            exit 1'
+  - '        fi'
+  - '        echo "Waiting 60 seconds before next attempt..."'
+  - '        sleep 60'
+  - '    fi'
+  - done
+  - '    echo "Verifying GPG key..."'
+  - '    rpm --import https://rpm.grafana.com/gpg.key'
+  - '    rpm -qa gpg-pubkey* | xargs rpm -qi | grep -i grafana'
+  - else
+  - '    echo "Grafana package version ${TAG} not found in repository."'
+  - '    dnf repolist'
+  - '    dnf list available grafana*'
+  - '    exit 1'
+  - fi
+  - 'echo "Step 6: Verifying Grafana installation..."'
+  - if rpm -q grafana | grep -q "${TAG}"; then
+  - '    echo "Successfully verified Grafana version ${TAG}"'
+  - else
+  - '    echo "Failed to verify Grafana version ${TAG}"'
+  - '    exit 1'
+  - fi
+  - echo "Verification complete."
+  depends_on:
+  - publish-linux-packages-rpm
+  environment: {}
+  image: rockylinux:9
+  name: verify-linux-RPM-packages
 - commands:
   - ./bin/build publish grafana-com --edition oss ${DRONE_TAG}
   depends_on:
@@ -4915,6 +5140,7 @@ steps:
   - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM cypress/included:13.10.0
   - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM jwilder/dockerize:0.6.1
   - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM koalaman/shellcheck:stable
+  - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM rockylinux:9
   depends_on:
   - authenticate-gcr
   image: aquasec/trivy:0.21.0
@@ -4950,6 +5176,7 @@ steps:
   - trivy --exit-code 1 --severity HIGH,CRITICAL cypress/included:13.10.0
   - trivy --exit-code 1 --severity HIGH,CRITICAL jwilder/dockerize:0.6.1
   - trivy --exit-code 1 --severity HIGH,CRITICAL koalaman/shellcheck:stable
+  - trivy --exit-code 1 --severity HIGH,CRITICAL rockylinux:9
   depends_on:
   - authenticate-gcr
   environment:
@@ -5181,6 +5408,6 @@ kind: secret
 name: gcr_credentials
 ---
 kind: signature
-hmac: 33b84712df805ae55115bdfedc6c40f71c75e7d6065656b49295b0f78f47bb9d
+hmac: 1824597b60bd1bf6f67ca7118c1fc3bd79271749b104e174ced460cbdaf26a5f
 
 ...

scripts/drone/events/release.star

@@ -22,6 +22,8 @@ load(
     "verify_gen_cue_step",
     "verify_gen_jsonnet_step",
     "verify_grafanacom_step",
+    "verify_linux_DEB_packages_step",
+    "verify_linux_RPM_packages_step",
     "wire_install_step",
     "yarn_install_step",
 )
@@ -203,6 +205,8 @@ def publish_packages_pipeline():
         compile_build_cmd(),
         publish_linux_packages_step(package_manager = "deb"),
         publish_linux_packages_step(package_manager = "rpm"),
+        verify_linux_DEB_packages_step(depends_on = ["publish-linux-packages-deb"]),
+        verify_linux_RPM_packages_step(depends_on = ["publish-linux-packages-rpm"]),
         publish_grafanacom_step(ver_mode = "release"),
         verify_grafanacom_step(),
     ]
@@ -223,6 +227,17 @@ def publish_packages_pipeline():
                 verify_grafanacom_step(depends_on = []),
             ],
         ),
+        pipeline(
+            name = "verify-linux-packages",
+            trigger = {
+                "event": ["promote"],
+                "target": "verify-linux-packages",
+            },
+            steps = [
+                verify_linux_DEB_packages_step(),
+                verify_linux_RPM_packages_step(),
+            ],
+        ),
         pipeline(
             name = "publish-packages",
             trigger = trigger,

scripts/drone/steps/lib.star

@@ -1241,6 +1241,110 @@ def publish_linux_packages_step(package_manager = "deb"):
         },
     }
 
+def retry_command(command, attempts = 5, delay = 60):
+    return [
+        "for i in $(seq 1 %d); do" % attempts,
+        "    if %s; then" % command,
+        '        echo "Command succeeded on attempt $i"',
+        "        break",
+        "    else",
+        '        echo "Attempt $i failed"',
+        "        if [ $i -eq %d ]; then" % attempts,
+        "            echo 'All attempts failed'",
+        "            exit 1",
+        "        fi",
+        '        echo "Waiting %d seconds before next attempt..."' % delay,
+        "        sleep %d" % delay,
+        "    fi",
+        "done",
+    ]
+
+def verify_linux_DEB_packages_step(depends_on = []):
+    install_command = "apt-get update >/dev/null 2>&1 && DEBIAN_FRONTEND=noninteractive apt-get install -yq grafana=${TAG} >/dev/null 2>&1"
+
+    return {
+        "name": "verify-linux-DEB-packages",
+        "image": images["ubuntu"],
+        "environment": {},
+        "commands": [
+            'echo "Step 1: Updating package lists..."',
+            "apt-get update >/dev/null 2>&1",
+            'echo "Step 2: Installing prerequisites..."',
+            "DEBIAN_FRONTEND=noninteractive apt-get install -yq apt-transport-https software-properties-common wget >/dev/null 2>&1",
+            'echo "Step 3: Adding Grafana GPG key..."',
+            "mkdir -p /etc/apt/keyrings/",
+            "wget -q -O - https://apt.grafana.com/gpg.key | gpg --dearmor | tee /etc/apt/keyrings/grafana.gpg > /dev/null",
+            'echo "Step 4: Adding Grafana repository..."',
+            'echo "deb [signed-by=/etc/apt/keyrings/grafana.gpg] https://apt.grafana.com stable main" | tee -a /etc/apt/sources.list.d/grafana.list',
+            'echo "Step 5: Installing Grafana..."',
+            # The packages take a bit of time to propogate within the repo. This retry will check their availability within 10 minutes.
+        ] + retry_command(install_command, attempts = 10) + [
+            'echo "Step 6: Verifying Grafana installation..."',
+            'if dpkg -s grafana | grep -q "Version: ${TAG}"; then',
+            '    echo "Successfully verified Grafana version ${TAG}"',
+            "else",
+            '    echo "Failed to verify Grafana version ${TAG}"',
+            "    exit 1",
+            "fi",
+            'echo "Verification complete."',
+        ],
+        "depends_on": depends_on,
+    }
+
+def verify_linux_RPM_packages_step(depends_on = []):
+    repo_config = (
+        "[grafana]\n" +
+        "name=grafana\n" +
+        "baseurl=https://rpm.grafana.com\n" +
+        "repo_gpgcheck=0\n" +  # Change this to 0
+        "enabled=1\n" +
+        "gpgcheck=0\n" +  # Change this to 0
+        "gpgkey=https://rpm.grafana.com/gpg.key\n" +
+        "sslverify=1\n" +
+        "sslcacert=/etc/pki/tls/certs/ca-bundle.crt\n"
+    )
+
+    repo_install_command = "dnf install -y --nogpgcheck grafana-${TAG} >/dev/null 2>&1"
+
+    return {
+        "name": "verify-linux-RPM-packages",
+        "image": images["rocky"],
+        "environment": {},
+        "commands": [
+            'echo "Step 1: Updating package lists..."',
+            "dnf check-update -y >/dev/null 2>&1 || true",
+            'echo "Step 2: Installing prerequisites..."',
+            "dnf install -y dnf-utils >/dev/null 2>&1",
+            'echo "Step 3: Adding Grafana GPG key..."',
+            "rpm --import https://rpm.grafana.com/gpg.key",
+            'echo "Step 4: Configuring Grafana repository..."',
+            "echo '" + repo_config + "' > /etc/yum.repos.d/grafana.repo",
+            'echo "Step 5: Checking RPM repository..."',
+            "dnf list available grafana-${TAG}",
+            "if [ $? -eq 0 ]; then",
+            '    echo "Grafana package found in repository. Installing from repo..."',
+        ] + retry_command(repo_install_command, attempts = 5) + [
+            '    echo "Verifying GPG key..."',
+            "    rpm --import https://rpm.grafana.com/gpg.key",
+            "    rpm -qa gpg-pubkey* | xargs rpm -qi | grep -i grafana",
+            "else",
+            '    echo "Grafana package version ${TAG} not found in repository."',
+            "    dnf repolist",
+            "    dnf list available grafana*",
+            "    exit 1",
+            "fi",
+            'echo "Step 6: Verifying Grafana installation..."',
+            'if rpm -q grafana | grep -q "${TAG}"; then',
+            '    echo "Successfully verified Grafana version ${TAG}"',
+            "else",
+            '    echo "Failed to verify Grafana version ${TAG}"',
+            "    exit 1",
+            "fi",
+            'echo "Verification complete."',
+        ],
+        "depends_on": depends_on,
+    }
+
 def verify_gen_cue_step():
     return {
         "name": "verify-gen-cue",

scripts/drone/utils/images.star

@@ -34,4 +34,5 @@ images = {
     "cypress": "cypress/included:13.10.0",
     "dockerize": "jwilder/dockerize:0.6.1",
     "shellcheck": "koalaman/shellcheck:stable",
+    "rocky": "rockylinux:9",
 }