diff --git a/devenv/docker/ha_test/alerts.sh b/devenv/docker/ha_test/alerts.sh
index a05a4581739..f2e7b4aec6c 100755
--- a/devenv/docker/ha_test/alerts.sh
+++ b/devenv/docker/ha_test/alerts.sh
@@ -101,7 +101,7 @@ provision() {
 
 	requiresJsonnet
 
-	rm -rf grafana/provisioning/dashboards/alerts/alert-*.json
+	find grafana/provisioning/dashboards/alerts -maxdepth 1 -name 'alert*.json' -delete
 	jsonnet -m grafana/provisioning/dashboards/alerts grafana/provisioning/alerts.jsonnet --ext-code alerts=$alerts --ext-code condition=$condition
 }