From 9fad55c87f0ae6d2d01c77d8c67c5883ccbdbfde Mon Sep 17 00:00:00 2001
From: Ed Santiago <santiago@redhat.com>
Date: Thu, 23 Jul 2020 10:54:50 -0600
Subject: [PATCH] CI - various fixes
Primary purpose: upgrade crun to 0.14 on f31, in hopes of
eliminating the 'cgroups.freeze' flake that is plaguing CI.
While I'm at it:
- remove a no-longer-needed dnf upgrade that was running in CI
itself (not image building, in each actual CI run). The purpose
was to upgrade conmon, but that was added a long time ago and
the required conmon is now in stable. The effect of this
dnf upgrade today was simply to cause flakes when fedora
repos were offline.
- remove a no-longer-needed check for varlink.
- networking.sh : add a timeout! 'openssl s_client' will happily
hang forever if a host is unreachable, which means we waste
two hours waiting for Cirrus to time out.
- timestamp.awk : include date (not just time) in START/END msgs.
There are times when I'm looking at a CI log and it is ultra
important to know if it is from yesterday or today.
- add progress messages in some places where I've previously
struggled to understand context in logs; and improve some
unlikely error messages to include script name.
...then, after all that, wrote a new README about how to to
all this. Hope it helps someone.
Signed-off-by: Ed Santiago <santiago@redhat.com>
---
.cirrus.yml | 2 +-
contrib/cirrus/check_image.sh | 3 -
contrib/cirrus/container_test.sh | 2 +
contrib/cirrus/integration_test.sh | 2 +-
contrib/cirrus/networking.sh | 2 +-
.../packer/README.how-to-update-cirrus-vms | 89 +++++++++++++++++++
contrib/cirrus/packer/fedora_packaging.sh | 9 ++
contrib/cirrus/rootless_test.sh | 4 +-
contrib/cirrus/setup_environment.sh | 3 -
contrib/cirrus/timestamp.awk | 4 +-
10 files changed, 108 insertions(+), 12 deletions(-)
create mode 100644 contrib/cirrus/packer/README.how-to-update-cirrus-vms
diff --git a/.cirrus.yml b/.cirrus.yml
index 37c9108eb0..fce9d2ac34 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -39,7 +39,7 @@ env:
UBUNTU_NAME: "ubuntu-20"
PRIOR_UBUNTU_NAME: "ubuntu-19"
- _BUILT_IMAGE_SUFFIX: "podman-5869602141896704"
+ _BUILT_IMAGE_SUFFIX: "podman-6439450735542272"
FEDORA_CACHE_IMAGE_NAME: "${FEDORA_NAME}-${_BUILT_IMAGE_SUFFIX}"
PRIOR_FEDORA_CACHE_IMAGE_NAME: "${PRIOR_FEDORA_NAME}-${_BUILT_IMAGE_SUFFIX}"
UBUNTU_CACHE_IMAGE_NAME: "${UBUNTU_NAME}-${_BUILT_IMAGE_SUFFIX}"
diff --git a/contrib/cirrus/check_image.sh b/contrib/cirrus/check_image.sh
index 0d33e55bfe..39c2be3f83 100755
--- a/contrib/cirrus/check_image.sh
+++ b/contrib/cirrus/check_image.sh
@@ -25,9 +25,6 @@ item_test 'Minimum available memory' $MEM_FREE -ge $MIN_MEM_MB || let "NFAILS+=1
remove_packaged_podman_files
item_test "remove_packaged_podman_files() does it's job" -z "$(type -P podman)" || let "NFAILS+=1"
-# Integration Tests require varlink in Fedora
-item_test "The varlink executable is present" -x "$(type -P varlink)" || let "NFAILS+=1"
-
MIN_ZIP_VER='3.0'
VER_RE='.+([[:digit:]]+\.[[:digit:]]+).+'
ACTUAL_VER=$(zip --version 2>&1 | egrep -m 1 "Zip$VER_RE" | sed -r -e "s/$VER_RE/\\1/")
diff --git a/contrib/cirrus/container_test.sh b/contrib/cirrus/container_test.sh
index bf0a0d3f11..f8d14c0e4d 100644
--- a/contrib/cirrus/container_test.sh
+++ b/contrib/cirrus/container_test.sh
@@ -18,6 +18,8 @@ if [ "${ID}" != "fedora" ] || [ "${CONTAINER_RUNTIME}" != "" ]; then
INTEGRATION_TEST_ENVS="SKIP_USERNS=1"
fi
+echo "$(date --rfc-3339=seconds) $(basename $0) started with '$*' and TEST_REMOTE_CLIENT='${TEST_REMOTE_CLIENT}'"
+
pwd
# -i install
diff --git a/contrib/cirrus/integration_test.sh b/contrib/cirrus/integration_test.sh
index 33e9fbc6b2..692d5a236b 100755
--- a/contrib/cirrus/integration_test.sh
+++ b/contrib/cirrus/integration_test.sh
@@ -16,7 +16,7 @@ fi
# but pr2947 intends to add 'system'.
TESTSUITE=$(expr $(basename $0) : '\(.*\)_test')
if [[ -z $TESTSUITE ]]; then
- die 1 "Script name is not of the form xxxx_test.sh"
+ die 1 "Script name ($basename $0) is not of the form xxxx_test.sh"
fi
cd "$GOSRC"
diff --git a/contrib/cirrus/networking.sh b/contrib/cirrus/networking.sh
index aeaf740352..2546fab71f 100755
--- a/contrib/cirrus/networking.sh
+++ b/contrib/cirrus/networking.sh
@@ -10,7 +10,7 @@ while read host port
do
if [[ "$port" -eq "443" ]]
then
- item_test "SSL/TLS to $host:$port" "$(echo -n '' | openssl s_client -quiet -no_ign_eof -connect $host:$port &> /dev/null; echo $?)" -eq "0"
+ item_test "SSL/TLS to $host:$port" "$(echo -n '' | timeout 60 openssl s_client -quiet -no_ign_eof -connect $host:$port &> /dev/null; echo $?)" -eq "0"
else
item_test "Connect to $host:$port" "$(nc -zv -w 13 $host $port &> /dev/null; echo $?)" -eq 0
fi
diff --git a/contrib/cirrus/packer/README.how-to-update-cirrus-vms b/contrib/cirrus/packer/README.how-to-update-cirrus-vms
new file mode 100644
index 0000000000..ac2902ffbc
--- /dev/null
+++ b/contrib/cirrus/packer/README.how-to-update-cirrus-vms
@@ -0,0 +1,89 @@
+This document briefly describes how to update VMs on Cirrus.
+
+Examples of when you need to do this:
+
+ - to update crun, conmon, or some other package(s)
+ - to add and/or remove an OS (eg drop f31, add f33)
+ - to change system config (eg containers.conf or other /etc files)
+ - to change kernel command-line (boot time) options
+
+This is a TWO-STEP process: you need to submit a PR with a magic [CI:IMG]
+description string, wait for it to finish, grab a magic string from the
+results, then resubmit without [CI:IMG].
+
+Procedure, Part One of Two:
+
+ 1) Create a working branch:
+
+ $ git co -b my_branch_name
+
+ 2) Make your changes. Typically, zero or more of the following files:
+
+ .cirrus.yml
+ contrib/cirrus/packer/*_packaging.sh
+
+ I said zero because sometimes you just want to update VMs
+ with the latest in dnf or ubuntu repos. That doesn't require
+ changing anything here, simply running new dnf/apt installs.
+
+ 3) Commit your changes. Be sure to include the magic [CI:IMG] string:
+
+ $ git commit -asm'[CI:IMG] this is my commit message'
+
+ 4) Submit your PR:
+
+ $ gh pr create --fill --web
+
+
+ -------------------------- INTERMISSION --------------------------
+ ...in which we wait for CI to turn green. In particular, although
+ we only really need 'test_build_cache_images' (45 minutes or so)
+ to get the required magic number strings, please be a decent
+ human being and wait for 'verify_test_built_images' (another hour)
+ so we can all have confidence in our process. Thank you.
+ -------------------------- INTERMISSION --------------------------
+
+
+Procedure, Part Two of Two:
+
+ 1) When 'test_build_cache_images' completes, click it, then click
+ 'View more details on Cirrus CI', then expand the 'Run build_vm_image'
+ accordion. This gives you a garishly colorful display of lines.
+ Each color is a different VM.
+
+ 2) Verify that each VM has the packages you require. (The garish log
+ doesn't actually list this for all packages, so you may need to
+ look in the 'verify_test_built_images' log for each individual
+ VM. Click the 'package_versions' accordion.)
+
+ 3) At the bottom of this log you will see a block like:
+
+ Builds finished. The artifacts of successful builds are:
+ ubuntu-19: A disk image was created: ubuntu-19-podman-6439450735542272
+ fedora-31: A disk image was created: fedora-31-podman-6439450735542272
+ .....
+
+ The long numbers at the end should (MUST!) be all identical.
+
+ 4) Edit .cirrus.yml locally. Find '_BUILT_IMAGE_SUFFIX' near the
+ top. Copy that long number ("6439450735542272", above) and paste
+ it here, replacing the previous long number.
+
+ 5) Wait for CI to turn green. I know you might have skipped that,
+ because 'test_build_cache_images' finishes long before 'verify',
+ and maybe you're in a hurry, but come on. Be responsible.
+
+ 6) Edit the PR description in github: remove '[CI:IMG]' from the
+ title. Again, *in github*, in the web UI, use the 'Edit' button
+ at top right next to the PR title. Remove the '[CI:IMG]' string
+ from the PR title, press Save. If you forget to do this, the
+ VM-building steps will run again (taking a long time) but it
+ will be a waste of time.
+
+ 7) Update your PR:
+
+ $ git add .cirrus.yml (to get the new magic IMAGE_SUFFIX string)
+ $ git commit --amend (remove [CI:IMG] for consistency with 6)
+ $ git push --force
+
+You can probably take it from here.
diff --git a/contrib/cirrus/packer/fedora_packaging.sh b/contrib/cirrus/packer/fedora_packaging.sh
index b4a3a20628..f19932a9ff 100644
--- a/contrib/cirrus/packer/fedora_packaging.sh
+++ b/contrib/cirrus/packer/fedora_packaging.sh
@@ -153,6 +153,15 @@ DOWNLOAD_PACKAGES=(\
echo "Installing general build/test dependencies for Fedora '$OS_RELEASE_VER'"
$BIGTO ooe.sh $SUDO dnf install -y ${INSTALL_PACKAGES[@]}
+# AD-HOC CODE FOR SPECIAL-CASE SITUATIONS!
+# On 2020-07-23 we needed this code to upgrade crun on f31, a build
+# that is not yet in stable. Since CI:IMG PRs are a two-step process,
+# the key part is that we UN-COMMENT-THIS-OUT during the first step,
+# then re-comment it on the second (once we have the built images).
+# That way this will be dead code in future CI:IMG PRs but will
+# serve as an example for anyone in a similar future situation.
+# $BIGTO ooe.sh $SUDO dnf --enablerepo=updates-testing -y upgrade crun
+
[[ ${#REMOVE_PACKAGES[@]} -eq 0 ]] || \
$LILTO ooe.sh $SUDO dnf erase -y ${REMOVE_PACKAGES[@]}
diff --git a/contrib/cirrus/rootless_test.sh b/contrib/cirrus/rootless_test.sh
index 9e1b1d911e..63cbec69b4 100755
--- a/contrib/cirrus/rootless_test.sh
+++ b/contrib/cirrus/rootless_test.sh
@@ -2,11 +2,13 @@
set -e
+echo "$(date --rfc-3339=seconds) $(basename $0) started with '$*'"
+
source $(dirname $0)/lib.sh
if [[ "$UID" == "0" ]]
then
- echo "Error: Expected to be running as a regular user"
+ echo "$(basename $0): Error: Expected to be running as a regular user"
exit 1
fi
diff --git a/contrib/cirrus/setup_environment.sh b/contrib/cirrus/setup_environment.sh
index fbdae83fa4..437a83c4b6 100755
--- a/contrib/cirrus/setup_environment.sh
+++ b/contrib/cirrus/setup_environment.sh
@@ -57,9 +57,6 @@ case "${OS_RELEASE_ID}" in
workaround_bfq_bug
- # HACK: Need Conmon 2.0.17, currently in updates-testing on F31.
- dnf update -y --enablerepo=updates-testing conmon
-
if [[ "$ADD_SECOND_PARTITION" == "true" ]]; then
bash "$SCRIPT_BASE/add_second_partition.sh"
fi
diff --git a/contrib/cirrus/timestamp.awk b/contrib/cirrus/timestamp.awk
index 95b312e515..b3663b303f 100644
--- a/contrib/cirrus/timestamp.awk
+++ b/contrib/cirrus/timestamp.awk
@@ -7,7 +7,7 @@
BEGIN {
STARTTIME=systime()
printf "[%s] START", strftime("%T")
- printf " - All [+xxxx] lines that follow are relative to right now.\n"
+ printf " - All [+xxxx] lines that follow are relative to %s.\n", strftime("%FT%T")
}
{
@@ -16,5 +16,5 @@ BEGIN {
END {
printf "[%s] END", strftime("%T")
- printf " - [%+05ds] total duration since START\n", systime()-STARTTIME
+ printf " - [%+05ds] total duration since %s\n", systime()-STARTTIME, strftime("%FT%T")
}