Merge pull request #7711 from cevich/migrate_imgs

Migrate container images to automation_images

.cirrus.yml

@@ -34,16 +34,16 @@ env:
     ####
     #### Cache-image names to test with (double-quotes around names are critical)
     ###
-    FEDORA_NAME: "fedora"
+    FEDORA_NAME: "fedora-32"
-    PRIOR_FEDORA_NAME: "prior-fedora"
+    PRIOR_FEDORA_NAME: "fedora-31"
-    UBUNTU_NAME: "ubuntu"
+    UBUNTU_NAME: "ubuntu-20"
-    PRIOR_UBUNTU_NAME: "prior-ubuntu"
+    PRIOR_UBUNTU_NAME: "ubuntu-19"
 
-    _BUILT_IMAGE_SUFFIX: "c6110627968057344"
+    _BUILT_IMAGE_SUFFIX: "c4948709391728640"
-    FEDORA_CACHE_IMAGE_NAME: "${FEDORA_NAME}-${_BUILT_IMAGE_SUFFIX}"
+    FEDORA_CACHE_IMAGE_NAME: "fedora-${_BUILT_IMAGE_SUFFIX}"
-    PRIOR_FEDORA_CACHE_IMAGE_NAME: "${PRIOR_FEDORA_NAME}-${_BUILT_IMAGE_SUFFIX}"
+    PRIOR_FEDORA_CACHE_IMAGE_NAME: "prior-fedora-${_BUILT_IMAGE_SUFFIX}"
-    UBUNTU_CACHE_IMAGE_NAME: "${UBUNTU_NAME}-${_BUILT_IMAGE_SUFFIX}"
+    UBUNTU_CACHE_IMAGE_NAME: "ubuntu-${_BUILT_IMAGE_SUFFIX}"
-    PRIOR_UBUNTU_CACHE_IMAGE_NAME: "${PRIOR_UBUNTU_NAME}-${_BUILT_IMAGE_SUFFIX}"
+    PRIOR_UBUNTU_CACHE_IMAGE_NAME: "prior-ubuntu-${_BUILT_IMAGE_SUFFIX}"
 
     ####
     #### Default to NOT operating in any special-case testing mode
@@ -290,7 +290,7 @@ build_without_cgo_task:
 meta_task:
 
     container:
-        image: "quay.io/libpod/imgts:master"  # see contrib/imgts
+        image: "quay.io/libpod/imgts:${_BUILT_IMAGE_SUFFIX}"
         cpu: 1
         memory: 1
 
@@ -301,7 +301,6 @@ meta_task:
             ${PRIOR_FEDORA_CACHE_IMAGE_NAME}
             ${UBUNTU_CACHE_IMAGE_NAME}
             ${PRIOR_UBUNTU_CACHE_IMAGE_NAME}
-            ${IMAGE_BUILDER_CACHE_IMAGE_NAME}
         BUILDID: "${CIRRUS_BUILD_ID}"
         REPOREF: "${CIRRUS_CHANGE_IN_REPO}"
         GCPJSON: ENCRYPTED[3a198350077849c8df14b723c0f4c9fece9ebe6408d35982e7adf2105a33f8e0e166ed3ed614875a0887e1af2b8775f4]

contrib/imgprune/Dockerfile

@@ -1,7 +0,0 @@
-FROM quay.io/libpod/imgts:latest
-
-RUN yum -y update && \
-    yum clean all
-
-COPY /contrib/imgprune/entrypoint.sh /usr/local/bin/entrypoint.sh
-RUN chmod 755 /usr/local/bin/entrypoint.sh

contrib/imgprune/README.md

@@ -1,11 +0,0 @@
-![PODMAN logo](../../logo/podman-logo-source.svg)
-
-A container image for maintaining the collection of
-VM images used by CI/CD on this project and several others.
-Acts upon metadata maintained by the imgts container.
-
-Example build (from repository root):
-
-```bash
-sudo podman build -t $IMAGE_NAME -f contrib/imgprune/Dockerfile .
-```

contrib/imgprune/entrypoint.sh

@@ -1,106 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-source /usr/local/bin/lib_entrypoint.sh
-
-req_env_var GCPJSON GCPNAME GCPPROJECT IMGNAMES
-
-unset BASE_IMAGES
-# When executing under Cirrus-CI, script have access to current source
-LIB="$CIRRUS_WORKING_DIR/$SCRIPT_BASE/lib.sh"
-if [[ "$CI" == "true" ]] && [[ -r "$LIB" ]]
-then
-    # Avoid importing anything that might conflict
-    for env in $(sed -ne 's/^[^#]\+_BASE_IMAGE=/img=/p' "$LIB")
-    do
-        eval $env
-        BASE_IMAGES="$BASE_IMAGES $img"
-    done
-else
-    # metadata labeling may have broken for some reason in the future
-    echo "Warning: Running outside of Cirrus-CI, very minor-risk of base-image deletion."
-fi
-
-gcloud_init
-
-# For safety's sake + limit nr background processes
-PRUNE_LIMIT=5
-THEFUTURE=$(date --date='+1 hour' +%s)
-TOO_OLD='30 days ago'
-THRESHOLD=$(date --date="$TOO_OLD" +%s)
-# Format Ref: https://cloud.google.com/sdk/gcloud/reference/topic/formats
-FORMAT='value[quote](name,selfLink,creationTimestamp,labels)'
-PROJRE="/v1/projects/$GCPPROJECT/global/"
-RECENTLY=$(date --date='3 days ago' --iso-8601=date)
-# Filter Ref: https://cloud.google.com/sdk/gcloud/reference/topic/filters
-FILTER="selfLink~$PROJRE AND creationTimestamp<$RECENTLY AND NOT name=($IMGNAMES $BASE_IMAGES)"
-TODELETE=$(mktemp -p '' todelete.XXXXXX)
-IMGCOUNT=$(mktemp -p '' imgcount.XXXXXX)
-
-# Search-loop runs in a sub-process, must store count in file
-echo "0" > "$IMGCOUNT"
-count_image() {
-    local count
-    count=$(<"$IMGCOUNT")
-    let 'count+=1'
-    echo "$count" > "$IMGCOUNT"
-}
-
-echo "Using filter: $FILTER"
-echo "Searching images for pruning candidates older than $TOO_OLD ($(date --date="$TOO_OLD" --iso-8601=date)):"
-$GCLOUD compute images list --format="$FORMAT" --filter="$FILTER" | \
-    while read name selfLink creationTimestamp labels
-    do
-        count_image
-        created_ymd=$(date --date=$creationTimestamp --iso-8601=date)
-        last_used=$(egrep --only-matching --max-count=1 'last-used=[[:digit:]]+' <<< $labels || true)
-        markmsgpfx="Marking $name (created $created_ymd) for deletion"
-        if [[ -z "$last_used" ]]
-        then # image pre-dates addition of tracking labels
-            echo "$markmsgpfx: Missing 'last-used' metadata, labels: '$labels'"
-            echo "$name" >> $TODELETE
-            continue
-        fi
-
-        last_used_timestamp=$(date --date=@$(cut -d= -f2 <<< $last_used || true) +%s || true)
-        last_used_ymd=$(date --date=@$last_used_timestamp --iso-8601=date)
-        if [[ -z "$last_used_timestamp" ]] || [[ "$last_used_timestamp" -ge "$THEFUTURE" ]]
-        then
-            echo "$markmsgpfx: Missing or invalid last-used timestamp: '$last_used_timestamp'"
-            echo "$name" >> $TODELETE
-            continue
-        fi
-
-        if [[ "$last_used_timestamp" -le "$THRESHOLD" ]]
-        then
-            echo "$markmsgpfx: Used over $TOO_OLD on $last_used_ymd"
-            echo "$name" >> $TODELETE
-            continue
-        fi
-    done
-
-COUNT=$(<"$IMGCOUNT")
-echo "########################################################################"
-echo "Deleting up to $PRUNE_LIMIT images marked ($(wc -l < $TODELETE)) of all searched ($COUNT):"
-
-# Require a minimum number of images to exist
-NEED="$[$PRUNE_LIMIT*2]"
-if [[ "$COUNT" -lt "$NEED" ]]
-then
-    die 0 Safety-net Insufficient images \($COUNT\) to process deletions \($NEED\)
-    exit 0
-fi
-
-for image_name in $(sort --random-sort $TODELETE | tail -$PRUNE_LIMIT)
-do
-    if echo "$IMGNAMES $BASE_IMAGES" | grep -q "$image_name"
-    then
-        # double-verify in-use images were filtered out in search loop above
-        die 8 FATAL ATTEMPT TO DELETE IN-USE IMAGE \'$image_name\' - THIS SHOULD NEVER HAPPEN
-    fi
-    echo "Deleting $image_name in parallel..."
-    $GCLOUD compute images delete $image_name &
-done
-
-wait || true  # Nothing to delete: No background jobs

contrib/imgts/Dockerfile

@@ -1,20 +0,0 @@
-FROM centos:7
-
-# Only needed for installing build-time dependencies
-COPY /contrib/imgts/google-cloud-sdk.repo /etc/yum.repos.d/google-cloud-sdk.repo
-RUN yum -y update && \
-    yum -y install epel-release && \
-    yum -y install google-cloud-sdk && \
-    yum clean all
-
-ENV GCPJSON="__unknown__" \
-    GCPNAME="__unknown__" \
-    GCPPROJECT="__unknown__" \
-    IMGNAMES="__unknown__" \
-    BUILDID="__unknown__" \
-    REPOREF="__unknown__"
-
-COPY ["/contrib/imgts/entrypoint.sh", "/contrib/imgts/lib_entrypoint.sh", "/usr/local/bin/"]
-RUN chmod 755 /usr/local/bin/entrypoint.sh
-
-ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]

contrib/imgts/README.md

@@ -1,11 +0,0 @@
-![PODMAN logo](../../logo/podman-logo-source.svg)
-
-A container image for tracking automation metadata.
-Currently this is used to update last-used timestamps on
-VM images.
-
-Example build (from repository root):
-
-```bash
-sudo podman build -t $IMAGE_NAME -f contrib/imgts/Dockerfile .
-```

contrib/imgts/entrypoint.sh

@@ -1,23 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-source /usr/local/bin/lib_entrypoint.sh
-
-req_env_var GCPJSON GCPNAME GCPPROJECT IMGNAMES BUILDID REPOREF
-
-gcloud_init
-
-ARGS="
-    --update-labels=last-used=$(date +%s)
-    --update-labels=build-id=$BUILDID
-    --update-labels=repo-ref=$REPOREF
-    --update-labels=project=$GCPPROJECT
-"
-
-for image in $IMGNAMES
-do
-    $GCLOUD compute images update "$image" $ARGS &
-done
-
-wait || echo "Warning: No \$IMGNAMES were specified."

contrib/imgts/google-cloud-sdk.repo

@@ -1,8 +0,0 @@
-[google-cloud-sdk]
-name=Google Cloud SDK
-baseurl=https://packages.cloud.google.com/yum/repos/cloud-sdk-el7-x86_64
-enabled=1
-gpgcheck=1
-repo_gpgcheck=1
-gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg
-       https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg

contrib/imgts/lib_entrypoint.sh

@@ -1,49 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-RED="\e[1;36;41m"
-YEL="\e[1;33;44m"
-NOR="\e[0m"
-SENTINEL="__unknown__"  # default set in dockerfile
-# Disable all input prompts
-# https://cloud.google.com/sdk/docs/scripting-gcloud
-GCLOUD="gcloud --quiet"
-
-die() {
-    EXIT=$1
-    PFX=$2
-    shift 2
-    MSG="$@"
-    echo -e "${RED}${PFX}:${NOR} ${YEL}$MSG${NOR}"
-    [[ "$EXIT" -eq "0" ]] || exit "$EXIT"
-}
-
-# Pass in a list of one or more envariable names; exit non-zero with
-# helpful error message if any value is empty
-req_env_var() {
-    for i; do
-        if [[ -z "${!i}" ]]
-        then
-            die 1 FATAL entrypoint.sh requires \$$i to be non-empty.
-        elif [[ "${!i}" == "$SENTINEL" ]]
-        then
-            die 2 FATAL entrypoint.sh requires \$$i to be explicitly set.
-        fi
-    done
-}
-
-gcloud_init() {
-    set +xe
-    if [[ -n "$1" ]] && [[ -r "$1" ]]
-    then
-        TMPF="$1"
-    else
-        TMPF=$(mktemp -p '' .$(uuidgen)_XXXX.json)
-        trap "rm -f $TMPF &> /dev/null" EXIT
-        echo "$GCPJSON" > $TMPF
-    fi
-    $GCLOUD auth activate-service-account --project="$GCPPROJECT" --key-file="$TMPF" || \
-        die 5 FATAL auth
-    rm -f $TMPF &> /dev/null || true  # ignore any read-only error
-}

contrib/upldrel/Dockerfile

@@ -1,9 +0,0 @@
-FROM quay.io/libpod/imgts:latest
-
-RUN yum -y update && \
-    yum -y install unzip && \
-    rpm -V unzip && \
-    yum clean all
-
-COPY /contrib/upldrel/entrypoint.sh /usr/local/bin/entrypoint.sh
-RUN chmod 755 /usr/local/bin/entrypoint.sh

contrib/upldrel/README.md

@@ -1,9 +0,0 @@
-![PODMAN logo](../../logo/podman-logo-source.svg)
-
-A container image for canonical-naming and uploading of
-libpod and remote-client archives.  Only intended to ever
-be used by CI/CD, and depends heavily on an embedded
-`release.txt` file produced by `make`.
-
-Build  script: [../cirrus/build_release.sh](../cirrus/build_release.sh)
-Upload script: [../cirrus/upload_release_archive.sh](../cirrus/upload_release_archive.sh)

contrib/upldrel/entrypoint.sh

@@ -1,27 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-source /usr/local/bin/lib_entrypoint.sh
-
-req_env_var GCPJSON_FILEPATH GCPNAME GCPPROJECT BUCKET FROM_FILEPATH TO_FILENAME
-
-[[ -r "$FROM_FILEPATH" ]] || \
-    die 2 ERROR Cannot read release archive file: "$FROM_FILEPATH"
-
-[[ -r "$GCPJSON_FILEPATH" ]] || \
-    die 3 ERROR Cannot read GCP credentials file: "$GCPJSON_FILEPATH"
-
-echo "Authenticating to google cloud for upload"
-gcloud_init "$GCPJSON_FILEPATH"
-
-echo "Uploading archive as $TO_FILENAME"
-gsutil cp "$FROM_FILEPATH" "gs://$BUCKET/$TO_FILENAME"
-[[ -z "$ALSO_FILENAME" ]] || \
-    gsutil cp "$FROM_FILEPATH" "gs://$BUCKET/$ALSO_FILENAME"
-
-echo "."
-echo "Release now available for download at:"
-echo "    https://storage.googleapis.com/$BUCKET/$TO_FILENAME"
-[[ -z "$ALSO_FILENAME" ]] || \
-    echo "    https://storage.googleapis.com/$BUCKET/$ALSO_FILENAME"