Cirrus: Add RHEL-7 testing

Add necessary tweaks to base-image build/import process so that image
can successfully boot with networking enabled.  Build the base image and
update ``.cirrus.yml`` accordingly.

Also make a minor improvement to cache-image building to help save space in
GCP image storage.  Namely, instead of storing every produced image, export
them into google storage buckets w/ life-cycle enabled.  For production use,
they may be converted back into GCE images, otherwise they will expire after
a time.

Signed-off-by: Chris Evich <cevich@redhat.com>

.cirrus.yml

@@ -31,7 +31,7 @@ env:
     PRIOR_FEDORA_CACHE_IMAGE_NAME: "fedora-28-libpod-7f4cd1f7"
     UBUNTU_CACHE_IMAGE_NAME: "ubuntu-18-libpod-7f4cd1f7"
     # RHEL_CACHE_IMAGE_NAME: "rhel-8-notready"
-    # PRIOR_RHEL_CACHE_IMAGE_NAME: "rhel-7-libpod-7f4cd1f7"
+    PRIOR_RHEL_CACHE_IMAGE_NAME: "rhel-7-libpod-7f4cd1f7"
     # CENTOS_CACHE_IMAGE_NAME: "centos-7-notready"
 
     ####
@@ -45,9 +45,9 @@ env:
     CRIU_COMMIT: "c74b83cd49c00589c0c0468ba5fe685b67fdbd0a"
     RUNC_COMMIT: "25f3f893c86d07426df93b7aa172f33fdf093fbd"
     # CSV of cache-image names to build (see $PACKER_BASE/libpod_images.json)
-    PACKER_BUILDS: "ubuntu-18,fedora-29,fedora-28"  # TODO: fah-29,rhel-7,centos-7
+    PACKER_BUILDS: "ubuntu-18,fedora-29,fedora-28,rhel-7"  # TODO: rhel-8,centos-7
     # Version of packer to use
-    PACKER_VER: "1.3.1"
+    PACKER_VER: "1.3.2"
     # Special image w/ nested-libvirt + tools for creating new cache and base images
     IMAGE_BUILDER_CACHE_IMAGE_NAME: "image-builder-image-1541772081"
     # Google-maintained base-image names
@@ -58,7 +58,7 @@ env:
     PRIOR_FEDORA_BASE_IMAGE: "fedora-cloud-base-28-1-1-1544474897"
     FAH_BASE_IMAGE:  "fedora-atomichost-29-20181025-1-1541787861"
     # RHEL image must be imported, google bills extra for their native image.
-    RHEL_BASE_IMAGE: "rhel-guest-image-7-6-210-x86-64-qcow2-1541783972"
+    RHEL_BASE_IMAGE: "rhel-guest-image-7-6-210-x86-64-qcow2-1548099756"
 
     ####
     #### Credentials and other secret-sauces, decrypted at runtime when authorized.
@@ -134,6 +134,7 @@ build_each_commit_task:
         - git fetch --depth $CIRRUS_CLONE_DEPTH origin $CIRRUS_BASE_BRANCH
         - env GOPATH=/var/tmp/go/ make build-all-new-commits GIT_BASE_BRANCH=origin/$CIRRUS_BASE_BRANCH
 
+
 # This task does the unit and integration testing for every platform
 testing_task:
 
@@ -154,10 +155,10 @@ testing_task:
             image_name: "${FEDORA_CACHE_IMAGE_NAME}"
             image_name: "${PRIOR_FEDORA_CACHE_IMAGE_NAME}"
             image_name: "${UBUNTU_CACHE_IMAGE_NAME}"
+            image_name: "${PRIOR_RHEL_CACHE_IMAGE_NAME}"
 
             # TODO: tests fail
-            # image_name: "${RHEL_CACHE_IMAGE_NAME}
+            # image_name: "${RHEL_CACHE_IMAGE_NAME}"
-            # image_name: "${PRIOR_RHEL_CACHE_IMAGE_NAME}
             # image_name: "${CENTOS_CACHE_IMAGE_NAME}"
 
     timeout_in: 120m
@@ -191,9 +192,9 @@ optional_testing_task:
             image_name: "${FEDORA_CACHE_IMAGE_NAME}"
             image_name: "${PRIOR_FEDORA_CACHE_IMAGE_NAME}"
             image_name: "${UBUNTU_CACHE_IMAGE_NAME}"
+            image_name: "${PRIOR_RHEL_CACHE_IMAGE_NAME}"
             # TODO: Make these work (also build_images_task below)
-            # image_name: "${RHEL_CACHE_IMAGE_NAME}
+            # image_name: "${RHEL_CACHE_IMAGE_NAME}"
-            # image_name: "${PRIOR_RHEL_CACHE_IMAGE_NAME}
             # image_name: "${CENTOS_CACHE_IMAGE_NAME}"
 
     timeout_in: 60m

contrib/cirrus/build_vm_images.sh

@@ -42,15 +42,33 @@ then
     fi
 fi
 
-set -x
-
 cd "$GOSRC/$PACKER_BASE"
+
+# Separate PR-produced images from those produced on master.
+if [[ "${CIRRUS_BRANCH:-}" == "master" ]]
+then
+    POST_MERGE_BUCKET_SUFFIX="-master"
+else
+    POST_MERGE_BUCKET_SUFFIX=""
+fi
+
 make libpod_images \
     PACKER_BUILDS=$PACKER_BUILDS \
     PACKER_VER=$PACKER_VER \
     GOSRC=$GOSRC \
     SCRIPT_BASE=$SCRIPT_BASE \
     PACKER_BASE=$PACKER_BASE \
+    POST_MERGE_BUCKET_SUFFIX=$POST_MERGE_BUCKET_SUFFIX \
     BUILT_IMAGE_SUFFIX=$BUILT_IMAGE_SUFFIX
 
 record_timestamp "cache-image build end"
+
+# When successful, upload manifest of produced images using a filename unique
+# to this build.
+URI="gs://packer-import${POST_MERGE_BUCKET_SUFFIX}/manifest${BUILT_IMAGE_SUFFIX}.json"
+gsutil cp packer-manifest.json "$URI"
+
+echo "Finished."
+echo "Any tarball URI's referenced above at at $URI"
+echo "may be used to create VM images suitable for use in"
+echo ".cirrus.yml as values for the 'image_name' keys."

contrib/cirrus/packer/Makefile

@@ -3,7 +3,7 @@
 # builder name(s) from applicable YAML file,
 # e.g for names see libpod_images.yml
 
-PACKER_VER ?= 1.3.1
+PACKER_VER ?= 1.3.2
 PACKER_DIST_FILENAME := packer_${PACKER_VER}_linux_amd64.zip
 
 # Only needed for libpod_base_images target
@@ -11,6 +11,7 @@ TIMESTAMP := $(shell date +%s)
 GOSRC ?= $(shell realpath "./../../../")
 PACKER_BASE ?= contrib/cirrus/packer
 SCRIPT_BASE ?= contrib/cirrus
+POST_MERGE_BUCKET_SUFFIX ?=
 
 # For debugging nested-virt, use
 #TTYDEV := $(shell tty)
@@ -50,10 +51,6 @@ endif
 		-var PACKER_BASE=$(PACKER_BASE) \
 		-var SCRIPT_BASE=$(SCRIPT_BASE) \
 		libpod_images.json
-	@echo ""
-	@echo "Finished.  The images mentioned above, and in packer-manifest.json"
-	@echo "can be used in .cirrus.yml as values for the 'image_name' keys"
-	@echo ""
 
 cidata.ssh:
 	ssh-keygen -f $@ -P "" -q
@@ -100,9 +97,6 @@ endif
 		-var RHEL_IMAGE_FILE=$(RHEL_IMAGE_FILE) \
 		-var RHEL_CSUM_FILE=$(RHEL_CSUM_FILE) \
 	    -var 'RHSM_COMMAND=$(RHSM_COMMAND)' \
+		-var POST_MERGE_BUCKET_SUFFIX=$(POST_MERGE_BUCKET_SUFFIX) \
 		-only $(PACKER_BUILDS) \
 		libpod_base_images.json
-	@echo ""
-	@echo "Finished.  The images mentioned above, and in packer-manifest.json"
-	@echo "can be used in .cirrus.yml as values for the *_BASE_IMAGE keys."
-	@echo ""

contrib/cirrus/packer/centos_setup.sh

@@ -25,6 +25,7 @@ ooe.sh sudo yum -y update
 ooe.sh sudo yum -y install centos-release-scl epel-release
 
 ooe.sh sudo yum -y install \
+    PyYAML \
     atomic-registries \
     btrfs-progs-devel \
     bzip2 \

contrib/cirrus/packer/image-builder-image_base-setup.sh

@@ -45,10 +45,13 @@ ooe.sh sudo yum -y install \
     qemu-kvm-tools \
     qemu-user \
     rsync \
+    rng-tools \
     unzip \
     util-linux \
     vim
 
+sudo systemctl enable rngd
+
 sudo ln -s /usr/libexec/qemu-kvm /usr/bin/
 
 sudo tee /etc/modprobe.d/kvm-nested.conf <<EOF

contrib/cirrus/packer/libpod_base_images.yml

@@ -105,7 +105,7 @@ builders:
       ssh_username: 'root'
 
     - <<: *nested_virt
-      name: 'prior_fedora'
+      name: 'prior-fedora'
       iso_url: '{{user `PRIOR_FEDORA_IMAGE_URL`}}'
       iso_checksum_url: '{{user `PRIOR_FEDORA_CSUM_URL`}}'
 
@@ -161,7 +161,7 @@ provisioners:
 
 post-processors:
     - - type: "compress"
-        only: ['fedora', 'prior_fedora', 'fah', 'rhel']
+        only: ['fedora', 'prior-fedora', 'fah', 'rhel']
         output: '/tmp/{{build_name}}/disk.raw.tar.gz'
         format: '.tar.gz'
         compression_level: 9
@@ -171,12 +171,12 @@ post-processors:
         project_id: '{{user `GCP_PROJECT_ID`}}'
         account_file: '{{user `GOOGLE_APPLICATION_CREDENTIALS`}}'
         bucket: '{{user `XFERBUCKET`}}'
-        gcs_object_name: '{{build_name}}-{{user `TIMESTAMP`}}-{{uuid}}.tar.gz'
+        gcs_object_name: '{{build_name}}-{{user `TIMESTAMP`}}.tar.gz'
         image_name: "{{user `FEDORA_BASE_IMAGE_NAME`}}-{{user `TIMESTAMP`}}"
         image_description: 'Based on {{user `FEDORA_IMAGE_URL`}}'
         image_family: '{{user `FEDORA_BASE_IMAGE_NAME`}}'
       - <<: *gcp_import
-        only: ['prior_fedora']
+        only: ['prior-fedora']
         image_name: "{{user `PRIOR_FEDORA_BASE_IMAGE_NAME`}}-{{user `TIMESTAMP`}}"
         image_description: 'Based on {{user `PRIOR_FEDORA_IMAGE_URL`}}'
         image_family: '{{user `PRIOR_FEDORA_BASE_IMAGE_NAME`}}'

contrib/cirrus/packer/libpod_images.yml

@@ -29,6 +29,10 @@ variables:
     SERVICE_ACCOUNT: '{{env `SERVICE_ACCOUNT`}}'
     GOOGLE_APPLICATION_CREDENTIALS: '{{env `GOOGLE_APPLICATION_CREDENTIALS`}}'
 
+    # Used to separate images produced during PR testing from those
+    # produced from post-merge testing.  Must be empty for PR testing.
+    POST_MERGE_BUCKET_SUFFIX: ''
+
 # Don't leak sensitive values in error messages / output
 sensitive-variables:
     - 'GCE_SSH_USERNAME'
@@ -84,13 +88,18 @@ provisioners:
     - type: 'shell'
       script: '{{user `GOSRC`}}/{{user `PACKER_BASE`}}/{{split build_name "-" 0}}_setup.sh'
       environment_vars:
-          - 'SCRIPT_BASE={{user `SCRIPT_BASE`}}'
+          - 'GOSRC=/tmp/libpod'
           - 'CNI_COMMIT={{user `CNI_COMMIT`}}'
           - 'FEDORA_CNI_COMMIT={{user `FEDORA_CNI_COMMIT`}}'
           - 'CRIO_COMMIT={{user `CRIO_COMMIT`}}'
           - 'CRIU_COMMIT={{user `CRIU_COMMIT`}}'
           - 'RUNC_COMMIT={{user `RUNC_COMMIT`}}'
+          - 'SCRIPT_BASE={{user `SCRIPT_BASE`}}'
           - 'RHSM_COMMAND={{user `RHSM_COMMAND`}}'
 
 post-processors:
-    - - type: 'manifest'
+    # Store VM disk in GCP storage, where it will expire based on a defined
+    # lifecycle. This prevents GCE from filling with disused images.
+    - - type: 'googlecompute-export'
+        paths: ['gs://packer-import{{user `POST_MERGE_BUCKET_SUFFIX`}}/{{build_name}}{{user `BUILT_IMAGE_SUFFIX`}}.tar.gz']
+      - type: 'manifest'  # writes packer-manifest.json

contrib/cirrus/packer/rhel_base-setup.sh

@@ -16,6 +16,8 @@ req_env_var "
 
 install_ooe
 
+rhsm_enable
+
 echo "Setting up repos"
 # Frequently needed
 ooe.sh sudo yum -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
@@ -32,12 +34,15 @@ gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg
        https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg
 EOM
 
-rhsm_enable
+echo "Updating all packages"
+ooe.sh sudo yum -y update
 
 echo "Installing/removing packages"
-ooe.sh sudo yum -y install google-compute-engine google-compute-engine-oslogin
+ooe.sh sudo yum -y install rng-tools google-compute-engine google-compute-engine-oslogin
-ooe.sh sudo yum -y erase "cloud-init" "rh-amazon-rhui-client*" || true
+
+echo "Enabling critical services"
 ooe.sh sudo systemctl enable \
+    rngd \
     google-accounts-daemon \
     google-clock-skew-daemon \
     google-instance-setup \
@@ -47,6 +52,29 @@ ooe.sh sudo systemctl enable \
 
 rhel_exit_handler  # release subscription!
 
+echo "Configuring boot"
+cat << "EOF" | sudo tee /etc/default/grub
+GRUB_TIMEOUT=0
+GRUB_DISTRIBUTOR="$(sed 's, release .*$,,g' /etc/system-release)"
+GRUB_DEFAULT=saved
+GRUB_DISABLE_SUBMENU=true
+GRUB_TERMINAL="serial console"
+GRUB_SERIAL_COMMAND="serial --speed=38400"
+GRUB_CMDLINE_LINUX="crashkernel=auto console=ttyS0,38400n8"
+GRUB_DISABLE_RECOVERY="true"
+EOF
+sudo grub2-mkconfig -o /boot/grub2/grub.cfg
+
+echo "Configuring networking"
+ooe.sh sudo nmcli connection modify 'System eth0' 802-3-ethernet.mtu 1460
+ooe.sh sudo nmcli connection modify 'System eth0' connection.autoconnect yes
+ooe.sh sudo nmcli connection modify 'System eth0' connection.autoconnect-priority
+ooe.sh sudo nmcli connection modify 'System eth0' ipv4.method auto
+ooe.sh sudo nmcli connection modify 'System eth0' ipv4.dhcp-send-hostname yes
+ooe.sh sudo nmcli connection modify 'System eth0' ipv4.dhcp-timeout 0
+ooe.sh sudo nmcli connection modify 'System eth0' ipv4.never-default no
+ooe.sh /usr/bin/google_instance_setup
+
 rh_finalize
 
 echo "SUCCESS!"

contrib/cirrus/packer/rhel_setup.sh

@@ -31,6 +31,7 @@ ooe.sh sudo subscription-manager repos \
 ooe.sh sudo yum -y update
 
 ooe.sh sudo yum -y install \
+    PyYAML \
     atomic-registries \
     btrfs-progs-devel \
     bzip2 \
@@ -64,9 +65,11 @@ ooe.sh sudo yum -y install \
     protobuf-python \
     python \
     python2-future \
+    python2-pyyaml \
     python34-dateutil \
     python34-psutil \
     python34-pytoml \
+    python34-PyYAML \
     runc \
     skopeo-containers \
     unzip \