Merge pull request #6201 from cevich/update_ubuntu20

Cirrus: Update Ubuntu 18 to 20

.cirrus.yml

@@ -36,10 +36,10 @@ env:
     ###
     FEDORA_NAME: "fedora-32"
     PRIOR_FEDORA_NAME: "fedora-31"
-    UBUNTU_NAME: "ubuntu-19"
-    PRIOR_UBUNTU_NAME: "ubuntu-18"
+    UBUNTU_NAME: "ubuntu-20"
+    PRIOR_UBUNTU_NAME: "ubuntu-19"
 
-    _BUILT_IMAGE_SUFFIX: "libpod-6224667180531712"  # From the packer output of 'build_vm_images_script'
+    _BUILT_IMAGE_SUFFIX: "libpod-6268069335007232"  # From the packer output of 'build_vm_images_script'
     FEDORA_CACHE_IMAGE_NAME: "${FEDORA_NAME}-${_BUILT_IMAGE_SUFFIX}"
     PRIOR_FEDORA_CACHE_IMAGE_NAME: "${PRIOR_FEDORA_NAME}-${_BUILT_IMAGE_SUFFIX}"
     UBUNTU_CACHE_IMAGE_NAME: "${UBUNTU_NAME}-${_BUILT_IMAGE_SUFFIX}"

Dockerfile.ubuntu

@@ -1,5 +1,5 @@
 # Must resemble $UBUNTU_BASE_IMAGE in ./contrib/cirrus/lib.sh
-FROM ubuntu:latest
+FROM ubuntu:20.04
 
 # This container image is intended for building and testing libpod
 # from inside a container environment.  It is assumed that the source

contrib/cirrus/lib.sh

@@ -63,11 +63,12 @@ CIRRUS_BUILD_ID=${CIRRUS_BUILD_ID:-$RANDOM$(date +%s)}  # must be short and uniq
 PACKER_VER="1.4.2"
 # CSV of cache-image names to build (see $PACKER_BASE/libpod_images.json)
 
-# Base-images rarely change, define them here so they're out of the way.
-export PACKER_BUILDS="${PACKER_BUILDS:-ubuntu-18,ubuntu-19,fedora-32,fedora-31}"
-# Manually produced base-image names (see $SCRIPT_BASE/README.md)
-export UBUNTU_BASE_IMAGE="ubuntu-1910-eoan-v20200211"
-export PRIOR_UBUNTU_BASE_IMAGE="ubuntu-1804-bionic-v20200218"
+# List of cache imaes to build for 'CI:IMG' mode via build_vm_images.sh
+# Exists to support manual single-image building in case of emergency
+export PACKER_BUILDS="${PACKER_BUILDS:-ubuntu-20,ubuntu-19,fedora-32,fedora-31}"
+# Google cloud provides these, we just make copies (see $SCRIPT_BASE/README.md) for use
+export UBUNTU_BASE_IMAGE="ubuntu-2004-focal-v20200506"
+export PRIOR_UBUNTU_BASE_IMAGE="ubuntu-1910-eoan-v20200211"
 # Manually produced base-image names (see $SCRIPT_BASE/README.md)
 export FEDORA_BASE_IMAGE="fedora-cloud-base-32-1-6-1588257430"
 export PRIOR_FEDORA_BASE_IMAGE="fedora-cloud-base-31-1-9-1588257430"

contrib/cirrus/packer/libpod_images.yml

@@ -29,7 +29,7 @@ sensitive-variables:
 builders:
     #  v----- is a YAML anchor, allows referencing this object by name (below)
     - &gce_hosted_image
-      name: 'ubuntu-19'
+      name: 'ubuntu-20'
       type: 'googlecompute'
       image_name: '{{build_name}}{{user `BUILT_IMAGE_SUFFIX`}}'
       image_family: '{{build_name}}-cache'
@@ -46,7 +46,7 @@ builders:
 
     #      v----- is a YAML alias, allows partial re-use of the anchor object
     - <<: *gce_hosted_image
-      name: 'ubuntu-18'
+      name: 'ubuntu-19'
       source_image: '{{user `PRIOR_UBUNTU_BASE_IMAGE`}}'
       source_image_family: 'prior-ubuntu-base'
 

contrib/cirrus/packer/ubuntu_packaging.sh

@@ -26,12 +26,6 @@ source /usr/share/automation/environment
 
 $LILTO ooe.sh $SUDOAPTADD ppa:criu/ppa
 
-# Install newer version of golang
-if [[ "$OS_RELEASE_VER" -eq "18" ]]
-then
-    $LILTO ooe.sh $SUDOAPTADD ppa:longsleep/golang-backports
-fi
-
 echo "Configuring/Instaling deps from Open build server"
 VERSION_ID=$(source /etc/os-release; echo $VERSION_ID)
 echo "deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_$VERSION_ID/ /" \
@@ -45,7 +39,9 @@ INSTALL_PACKAGES=(\
     autoconf
     automake
     bash-completion
+    bats
     bison
+    btrfs-progs
     build-essential
     buildah
     bzip2
@@ -60,6 +56,7 @@ INSTALL_PACKAGES=(\
     e2fslibs-dev
     emacs-nox
     file
+    fuse3
     gawk
     gcc
     gettext
@@ -71,11 +68,13 @@ INSTALL_PACKAGES=(\
     jq
     libaio-dev
     libapparmor-dev
+    libbtrfs-dev
     libcap-dev
     libdevmapper-dev
     libdevmapper1.02.1
     libfuse-dev
     libfuse2
+    libfuse3-dev
     libglib2.0-dev
     libgpgme11-dev
     liblzma-dev
@@ -99,8 +98,6 @@ INSTALL_PACKAGES=(\
     podman
     protobuf-c-compiler
     protobuf-compiler
-    python-future
-    python-minimal
     python-protobuf
     python3-dateutil
     python3-pip
@@ -118,29 +115,16 @@ INSTALL_PACKAGES=(\
     vim
     wget
     xz-utils
-    yum-utils
     zip
     zlib1g-dev
 )
 
-if [[ $OS_RELEASE_VER -ge 19 ]]
-then
+# These aren't resolvable on Ubuntu 20
+if [[ "$OS_RELEASE_VER" -le 19 ]]; then
     INSTALL_PACKAGES+=(\
-        bats
-        btrfs-progs
-        fuse3
-        libbtrfs-dev
-        libfuse3-dev
-    )
-else
-    echo "Downloading version of bats with fix for a \$IFS related bug in 'run' command"
-    cd /tmp
-    BATS_URL='http://launchpadlibrarian.net/438140887/bats_1.1.0+git104-g1c83a1b-1_all.deb'
-    curl -L -O "$BATS_URL"
-    cd -
-    INSTALL_PACKAGES+=(\
-        /tmp/$(basename $BATS_URL)
-        btrfs-tools
+        python-future
+        python-minimal
+        yum-utils
     )
 fi
 

hack/get_ci_vm.sh

@@ -67,13 +67,6 @@ delvm() {
     cleanup
 }
 
-image_hints() {
-    _BIS=$(egrep -m 1 '_BUILT_IMAGE_SUFFIX:[[:space:]+"[[:print:]]+"' "$LIBPODROOT/.cirrus.yml" | cut -d: -f 2 | tr -d '"[:blank:]')
-    egrep '[[:space:]]+[[:alnum:]].+_CACHE_IMAGE_NAME:[[:space:]+"[[:print:]]+"' \
-        "$LIBPODROOT/.cirrus.yml" | cut -d: -f 2 | tr -d '"[:blank:]' | \
-        sed -r -e "s/\\\$[{]_BUILT_IMAGE_SUFFIX[}]/$_BIS/" | sort -u
-}
-
 show_usage() {
     echo -e "\n${RED}ERROR: $1${NOR}"
     echo -e "${YEL}Usage: $(basename $0) [-m <SPECIALMODE>] [-u <ROOTLESS_USER> ] <image_name>${NOR}"
@@ -90,17 +83,34 @@ show_usage() {
 }
 
 get_env_vars() {
-    python -c '
-import yaml
+    # Deal with both YAML and embedded shell-like substitutions in values
+    # if substitution fails, fall back to printing naked env. var as-is.
+    python3 -c '
+import yaml,re
 env=yaml.load(open(".cirrus.yml"), Loader=yaml.SafeLoader)["env"]
-keys=[k for k in env if "ENCRYPTED" not in str(env[k])]
+dollar_env_var=re.compile(r"\$(\w+)")
+dollarcurly_env_var=re.compile(r"\$\{(\w+)\}")
+class ReIterKey(dict):
+    def __missing__(self, key):
+        # Cirrus-CI provides some runtime-only env. vars.  Avoid
+        # breaking this hack-script if/when any are present in YAML
+        return "${0}".format(key)
+rep=r"{\1}"  # Convert env vars markup to -> str.format_map(re_iter_key) markup
+out=ReIterKey()
 for k,v in env.items():
     v=str(v)
-    if "ENCRYPTED" not in v and "ADD_SECOND_PARTITION" not in v:
-        print("{0}=\"{1}\"".format(k, v)),
+    if "ENCRYPTED" not in v:
+        out[k]=dollar_env_var.sub(rep, dollarcurly_env_var.sub(rep, v))
+for k,v in out.items():
+    print("{0}=\"{1}\"".format(k, v.format_map(out)))
     '
 }
 
+image_hints() {
+    get_env_vars | fgrep '_CACHE_IMAGE_NAME' | awk -F "=" '{print $2}'
+}
+
+
 parse_args(){
     echo -e "$USAGE_WARNING"