Merge pull request #19979 from edsantiago/ci_trace_setup

CI: trace setup and runner scripts
This commit is contained in:
OpenShift Merge Robot
2023-09-15 12:41:46 +02:00
committed by GitHub
3 changed files with 98 additions and 73 deletions

View File

@ -79,9 +79,11 @@ CIRRUS_REPO_NAME=${CIRRUS_REPO_NAME:-podman}
# shellcheck disable=SC2154
if [[ -z "$CIRRUS_BASE_SHA" ]] && [[ -z "$CIRRUS_TAG" ]]
then # Operating on a branch, or under `get_ci_vm.sh`
showrun echo "branch or get_ci_vm (CIRRUS_BASE_SHA and CIRRUS_TAG are unset)"
CIRRUS_BASE_SHA=$(git rev-parse ${UPSTREAM_REMOTE:-origin}/$DEST_BRANCH)
elif [[ -z "$CIRRUS_BASE_SHA" ]]
then # Operating on a tag
showrun echo "operating on tag"
CIRRUS_BASE_SHA=$(git rev-parse HEAD)
fi
# The starting place for linting and code validation
@ -164,8 +166,8 @@ setup_rootless() {
ROOTLESS_UID=$rootless_uid
rootless_gid=$((1500 + RANDOM % 5000))
msg "creating $rootless_uid:$rootless_gid $ROOTLESS_USER user"
groupadd -g $rootless_gid $ROOTLESS_USER
useradd -g $rootless_gid -u $rootless_uid --no-user-group --create-home $ROOTLESS_USER
showrun groupadd -g $rootless_gid $ROOTLESS_USER
showrun useradd -g $rootless_gid -u $rootless_uid --no-user-group --create-home $ROOTLESS_USER
echo "$ROOTLESS_USER ALL=(root) NOPASSWD: ALL" > /etc/sudoers.d/ci-rootless
@ -174,8 +176,8 @@ setup_rootless() {
msg "Creating ssh key pairs"
[[ -r "$HOME/.ssh/id_rsa" ]] || \
ssh-keygen -t rsa -P "" -f "$HOME/.ssh/id_rsa"
ssh-keygen -t ed25519 -P "" -f "/home/$ROOTLESS_USER/.ssh/id_ed25519"
ssh-keygen -t rsa -P "" -f "/home/$ROOTLESS_USER/.ssh/id_rsa"
showrun ssh-keygen -t ed25519 -P "" -f "/home/$ROOTLESS_USER/.ssh/id_ed25519"
showrun ssh-keygen -t rsa -P "" -f "/home/$ROOTLESS_USER/.ssh/id_rsa"
msg "Set up authorized_keys"
cat $HOME/.ssh/*.pub /home/$ROOTLESS_USER/.ssh/*.pub >> $HOME/.ssh/authorized_keys
@ -231,17 +233,17 @@ use_cni() {
[ -z "$(rpm -qa | grep $pkg)" ] && echo "$pkg not installed" || rpm -e --nodeps $pkg
done
msg "Installing default CNI configuration"
dnf install -y $PACKAGE_DOWNLOAD_DIR/podman-plugins*
showrun dnf install -y $PACKAGE_DOWNLOAD_DIR/podman-plugins*
cd $GOSRC || exit 1
rm -rvf /etc/cni/net.d
mkdir -p /etc/cni/net.d
install -v -D -m 644 ./cni/87-podman-bridge.conflist \
showrun install -v -D -m 644 ./cni/87-podman-bridge.conflist \
/etc/cni/net.d/
# This config must always sort last in the list of networks (podman picks
# first one as the default). This config prevents allocation of network
# address space used by default in google cloud.
# https://cloud.google.com/vpc/docs/vpc#ip-ranges
install -v -D -m 644 $SCRIPT_BASE/99-do-not-use-google-subnets.conflist \
showrun install -v -D -m 644 $SCRIPT_BASE/99-do-not-use-google-subnets.conflist \
/etc/cni/net.d/
}
@ -252,7 +254,7 @@ use_netavark() {
echo "NETWORK_BACKEND=netavark" >> /etc/ci_environment
export NETWORK_BACKEND=netavark # needed for install_test_configs()
msg "Removing any/all CNI configuration"
rm -rvf /etc/cni/net.d/*
showrun rm -rvf /etc/cni/net.d/*
# N/B: The CNI packages are still installed and available. This is
# on purpose, since CI needs to verify the selection mechanisms are
# functional when both are available.
@ -276,8 +278,8 @@ remove_packaged_podman_files() {
if systemctl --quiet is-$state $unit
then
echo "Warning: $unit found $state prior to packaged-file removal"
systemctl --quiet disable $unit || true
systemctl --quiet stop $unit || true
showrun systemctl --quiet disable $unit || true
showrun systemctl --quiet stop $unit || true
fi
done
done
@ -299,9 +301,11 @@ remove_packaged_podman_files() {
do
# Sub-directories may contain unrelated/valuable stuff
if [[ -d "$fullpath" ]]; then continue; fi
ooe.sh rm -vf "$fullpath"
showrun ooe.sh rm -vf "$fullpath"
done
# Be super extra sure and careful vs performant and completely safe
sync && echo 3 > /proc/sys/vm/drop_caches || true
}
showrun echo "finished"

View File

@ -19,15 +19,17 @@ set -eo pipefail
# shellcheck source=contrib/cirrus/lib.sh
source $(dirname $0)/lib.sh
showrun echo "starting"
function _run_validate() {
# TODO: aarch64 images need python3-devel installed
# https://github.com/containers/automation_images/issues/159
bigto ooe.sh dnf install -y python3-devel
showrun bigto ooe.sh dnf install -y python3-devel
# git-validation tool fails if $EPOCH_TEST_COMMIT is empty
# shellcheck disable=SC2154
if [[ -n "$EPOCH_TEST_COMMIT" ]]; then
make validate
showrun make validate
else
warn "Skipping git-validation since \$EPOCH_TEST_COMMIT is empty"
fi
@ -42,29 +44,29 @@ function _run_unit() {
# shellcheck disable=SC2154
die "$TEST_FLAVOR: Unsupported PODBIN_NAME='$PODBIN_NAME'"
fi
make localunit
showrun make localunit
}
function _run_apiv2() {
_bail_if_test_can_be_skipped test/apiv2
(
make localapiv2-bash
showrun make localapiv2-bash
source .venv/requests/bin/activate
make localapiv2-python
showrun make localapiv2-python
) |& logformatter
}
function _run_compose() {
_bail_if_test_can_be_skipped test/compose
./test/compose/test-compose |& logformatter
showrun ./test/compose/test-compose |& logformatter
}
function _run_compose_v2() {
_bail_if_test_can_be_skipped test/compose
./test/compose/test-compose |& logformatter
showrun ./test/compose/test-compose |& logformatter
}
function _run_int() {
@ -82,18 +84,18 @@ function _run_sys() {
function _run_upgrade_test() {
_bail_if_test_can_be_skipped test/system test/upgrade
bats test/upgrade |& logformatter
showrun bats test/upgrade |& logformatter
}
function _run_bud() {
_bail_if_test_can_be_skipped test/buildah-bud
./test/buildah-bud/run-buildah-bud-tests |& logformatter
showrun ./test/buildah-bud/run-buildah-bud-tests |& logformatter
}
function _run_bindings() {
# install ginkgo
make .install.ginkgo
showrun make .install.ginkgo
# shellcheck disable=SC2155
export PATH=$PATH:$GOSRC/hack:$GOSRC/test/tools/build
@ -105,23 +107,23 @@ function _run_bindings() {
fi
(echo "$gitcommit_magic" && \
make testbindings) |& logformatter
showrun make testbindings) |& logformatter
}
function _run_docker-py() {
source .venv/docker-py/bin/activate
make run-docker-py-tests
showrun make run-docker-py-tests
}
function _run_endpoint() {
make test-binaries
make endpoint
showrun make test-binaries
showrun make endpoint
}
function _run_minikube() {
_bail_if_test_can_be_skipped test/minikube
msg "Testing minikube."
bats test/minikube |& logformatter
showrun bats test/minikube |& logformatter
}
exec_container() {
@ -187,10 +189,10 @@ function _run_swagger() {
# Swagger validation takes a significant amount of time
msg "Pulling \$CTR_FQIN '$CTR_FQIN' (background process)"
bin/podman pull --quiet $CTR_FQIN &
showrun bin/podman pull --quiet $CTR_FQIN &
cd $GOSRC
make swagger
showrun make swagger
# Cirrus-CI Artifact instruction expects file here
cp -v $GOSRC/pkg/api/swagger.yaml ./
@ -209,7 +211,7 @@ eof
msg "Waiting for backgrounded podman pull to complete..."
wait %%
bin/podman run -it --rm --security-opt label=disable \
showrun bin/podman run -it --rm --security-opt label=disable \
--env-file=$envvarsfile \
-v $GOSRC:$GOSRC:ro \
--workdir $GOSRC \
@ -219,9 +221,9 @@ eof
function _run_build() {
# Ensure always start from clean-slate with all vendor modules downloaded
make clean
make vendor
make podman-release # includes podman, podman-remote, and docs
showrun make clean
showrun make vendor
showrun make podman-release # includes podman, podman-remote, and docs
# Last-minute confirmation that we're testing the desired runtime.
# This Can't Possibly Fail™ in regular CI; only when updating VMs.
@ -252,7 +254,7 @@ function _run_altbuild() {
cd $GOSRC
case "$ALT_NAME" in
*Each*)
git fetch origin
showrun git fetch origin
# The make-and-check-size script, introduced 2022-03-22 in #13518,
# runs 'make' (the original purpose of this check) against
# each commit, then checks image sizes to make sure that
@ -264,19 +266,19 @@ function _run_altbuild() {
savedhead=$(git rev-parse HEAD)
# Push to PR base. First run of the script will write size files
pr_base=$(git merge-base origin/$DEST_BRANCH HEAD)
git checkout $pr_base
hack/make-and-check-size $context_dir
showrun git checkout $pr_base
showrun hack/make-and-check-size $context_dir
# pop back to PR, and run incremental makes. Subsequent script
# invocations will compare against original size.
git checkout $savedhead
git rebase $pr_base -x "hack/make-and-check-size $context_dir"
showrun git checkout $savedhead
showrun git rebase $pr_base -x "hack/make-and-check-size $context_dir"
rm -rf $context_dir
;;
*Windows*)
make podman-remote-release-windows_amd64.zip
showrun make podman-remote-release-windows_amd64.zip
;;
*RPM*)
make package
showrun make package
;;
Alt*x86*Cross)
arches=(\
@ -316,7 +318,7 @@ function _run_altbuild() {
function _build_altbuild_archs() {
for arch in "$@"; do
msg "Building release archive for $arch"
make podman-release-${arch}.tar.gz GOARCH=$arch
showrun make podman-release-${arch}.tar.gz GOARCH=$arch
done
}
@ -418,13 +420,13 @@ dotest() {
die "Found fallback podman '$fallback_podman' in \$PATH; tests require none, as a guarantee that we're testing the right binary."
fi
make ${localremote}${testsuite} PODMAN_SERVER_LOG=$PODMAN_SERVER_LOG \
showrun make ${localremote}${testsuite} PODMAN_SERVER_LOG=$PODMAN_SERVER_LOG \
|& logformatter
}
_run_machine() {
# N/B: Can't use _bail_if_test_can_be_skipped here b/c content isn't under test/
make localmachine |& logformatter
showrun make localmachine |& logformatter
}
# Optimization: will exit if the only PR diffs are under docs/ or tests/
@ -543,4 +545,6 @@ if [ "$(type -t $handler)" != "function" ]; then
die "Unknown/Unsupported \$TEST_FLAVOR=$TEST_FLAVOR"
fi
$handler
showrun $handler
showrun echo "finished"

View File

@ -12,6 +12,8 @@ set -e
# shellcheck source=./contrib/cirrus/lib.sh
source $(dirname $0)/lib.sh
showrun echo "starting"
die_unknown() {
local var_name="$1"
req_env_vars var_name
@ -40,10 +42,10 @@ cp hack/podman-registry /bin
# Some test operations & checks require a git "identity"
_gc='git config --file /root/.gitconfig'
$_gc user.email "TMcTestFace@example.com"
$_gc user.name "Testy McTestface"
showrun $_gc user.email "TMcTestFace@example.com"
showrun $_gc user.name "Testy McTestface"
# Bypass git safety/security checks when operating in a throwaway environment
git config --system --add safe.directory $GOSRC
showrun git config --system --add safe.directory $GOSRC
# Ensure that all lower-level contexts and child-processes have
# ready access to higher level orchestration (e.g Cirrus-CI)
@ -82,6 +84,7 @@ mkdir -p /etc/containers/containers.conf.d
# respectively.
# **IMPORTANT**: $OCI_RUNTIME is a fakeout! It is used only in e2e tests.
# For actual podman, as in system tests, we force runtime in containers.conf
showrun echo "conditional check: CG_FS_TYPE [=$CG_FS_TYPE]"
case "$CG_FS_TYPE" in
tmpfs)
if ((CONTAINER==0)); then
@ -107,6 +110,7 @@ printf "[engine]\ndatabase_backend=\"$CI_DESIRED_DATABASE\"\n" > /etc/containers
# does not defaults to using `vfs` as storage driver)
# shellcheck disable=SC2154
if [[ "$OS_RELEASE_ID" == "debian" ]]; then
showrun echo "conditional setup for debian"
conf=/etc/containers/storage.conf
if [[ -e $conf ]]; then
die "FATAL! INTERNAL ERROR! Cannot override $conf"
@ -116,6 +120,7 @@ if [[ "$OS_RELEASE_ID" == "debian" ]]; then
fi
if ((CONTAINER==0)); then # Not yet running inside a container
showrun echo "conditional setup for CONTAINER == 0"
# Discovered reemergence of BFQ scheduler bug in kernel 5.8.12-200
# which causes a kernel panic when system is under heavy I/O load.
# Disable the I/O scheduler (a.k.a. elevator) for all environments,
@ -145,22 +150,24 @@ fi
# Which distribution are we testing on.
case "$OS_RELEASE_ID" in
debian)
showrun echo "more conditional setup for debian"
# FIXME 2023-04-11: workaround for runc regression causing failure
# in system tests: "skipping device /dev/char/10:200 for systemd"
# (Checked on 2023-08-08 and it's still too old: 1.1.5)
# FIXME: please remove this once runc >= 1.2 makes it into debian.
modprobe tun
showrun modprobe tun
# TODO: move this into image build process
# We need the "en_US.UTF-8" locale for the "podman logs with non ASCII log tag" tests
sed -i '/en_US.UTF-8/s/^#//g' /etc/locale.gen
locale-gen
showrun sed -i '/en_US.UTF-8/s/^#//g' /etc/locale.gen
showrun locale-gen
;;
fedora)
showrun echo "conditional setup for fedora"
if ((CONTAINER==0)); then
# All SELinux distros need this for systemd-in-a-container
msg "Enabling container_manage_cgroup"
setsebool container_manage_cgroup true
showrun setsebool container_manage_cgroup true
fi
;;
*) die_unknown OS_RELEASE_ID
@ -169,6 +176,7 @@ esac
# Networking: force CNI or Netavark as requested in .cirrus.yml
# (this variable is mandatory).
# shellcheck disable=SC2154
showrun echo "about to set up for CI_DESIRED_NETWORK [=$CI_DESIRED_NETWORK]"
case "$CI_DESIRED_NETWORK" in
netavark) use_netavark ;;
cni) use_cni ;;
@ -178,6 +186,7 @@ esac
# Database: force SQLite or BoltDB as requested in .cirrus.yml.
# If unset, will default to BoltDB.
# shellcheck disable=SC2154
showrun echo "about to set up for CI_DESIRED_DATABASE [=$CI_DESIRED_DATABASE]"
case "$CI_DESIRED_DATABASE" in
sqlite)
warn "Forcing PODMAN_DB=sqlite"
@ -197,6 +206,7 @@ esac
# Required to be defined by caller: The environment where primary testing happens
# shellcheck disable=SC2154
showrun echo "about to set up for TEST_ENVIRON [=$TEST_ENVIRON]"
case "$TEST_ENVIRON" in
host)
# The e2e tests wrongly guess `--cgroup-manager` option
@ -244,6 +254,7 @@ case "$TEST_ENVIRON" in
esac
# Required to be defined by caller: Are we testing as root or a regular user
showrun echo "about to set up for PRIV_NAME [=$PRIV_NAME]"
case "$PRIV_NAME" in
root)
# shellcheck disable=SC2154
@ -265,6 +276,7 @@ esac
# shellcheck disable=SC2154
if [[ -n "$ROOTLESS_USER" ]]; then
showrun echo "conditional setup for ROOTLESS_USER [=$ROOTLESS_USER]"
echo "ROOTLESS_USER=$ROOTLESS_USER" >> /etc/ci_environment
echo "ROOTLESS_UID=$ROOTLESS_UID" >> /etc/ci_environment
fi
@ -285,7 +297,7 @@ if ((CONTAINER==0)); then
nsswitch=/etc/authselect/nsswitch.conf
if [[ -e $nsswitch ]]; then
if grep -q -E 'hosts:.*resolve' $nsswitch; then
msg "Disabling systemd-resolved"
showrun echo "Disabling systemd-resolved"
sed -i -e 's/^\(hosts: *\).*/\1files dns myhostname/' $nsswitch
systemctl stop systemd-resolved
rm -f /etc/resolv.conf
@ -322,46 +334,47 @@ esac
# Required to be defined by caller: The primary type of testing that will be performed
# shellcheck disable=SC2154
showrun echo "about to set up for TEST_FLAVOR [=$TEST_FLAVOR]"
case "$TEST_FLAVOR" in
validate)
dnf install -y $PACKAGE_DOWNLOAD_DIR/python3*.rpm
showrun dnf install -y $PACKAGE_DOWNLOAD_DIR/python3*.rpm
# For some reason, this is also needed for validation
make .install.pre-commit .install.gitvalidation
showrun make .install.pre-commit .install.gitvalidation
;;
altbuild)
# Defined in .cirrus.yml
# shellcheck disable=SC2154
if [[ "$ALT_NAME" =~ RPM ]]; then
bigto dnf install -y glibc-minimal-langpack go-rpm-macros rpkg rpm-build shadow-utils-subid-devel
showrun bigto dnf install -y glibc-minimal-langpack go-rpm-macros rpkg rpm-build shadow-utils-subid-devel
fi
;;
docker-py)
remove_packaged_podman_files
make install PREFIX=/usr ETCDIR=/etc
showrun make install PREFIX=/usr ETCDIR=/etc
msg "Installing previously downloaded/cached packages"
dnf install -y $PACKAGE_DOWNLOAD_DIR/python3*.rpm
showrun dnf install -y $PACKAGE_DOWNLOAD_DIR/python3*.rpm
virtualenv .venv/docker-py
source .venv/docker-py/bin/activate
pip install --upgrade pip
pip install --requirement $GOSRC/test/python/requirements.txt
showrun pip install --upgrade pip
showrun pip install --requirement $GOSRC/test/python/requirements.txt
;;
build) make clean ;;
unit)
make .install.ginkgo
showrun make .install.ginkgo
;;
compose_v2)
dnf -y remove docker-compose
curl -SL https://github.com/docker/compose/releases/download/v2.2.3/docker-compose-linux-x86_64 -o /usr/local/bin/docker-compose
chmod +x /usr/local/bin/docker-compose
showrun dnf -y remove docker-compose
showrun curl -SL https://github.com/docker/compose/releases/download/v2.2.3/docker-compose-linux-x86_64 -o /usr/local/bin/docker-compose
showrun chmod +x /usr/local/bin/docker-compose
;& # Continue with next item
apiv2)
msg "Installing previously downloaded/cached packages"
dnf install -y $PACKAGE_DOWNLOAD_DIR/python3*.rpm
showrun dnf install -y $PACKAGE_DOWNLOAD_DIR/python3*.rpm
virtualenv .venv/requests
source .venv/requests/bin/activate
pip install --upgrade pip
pip install --requirement $GOSRC/test/apiv2/python/requirements.txt
showrun pip install --upgrade pip
showrun pip install --requirement $GOSRC/test/apiv2/python/requirements.txt
;& # continue with next item
compose)
showrun make install.tools
@ -369,13 +382,14 @@ case "$TEST_FLAVOR" in
showrun dnf install -y podman-docker*
;& # continue with next item
int)
make .install.ginkgo
showrun make .install.ginkgo
;&
sys) ;&
upgrade_test) ;&
bud) ;&
bindings) ;&
endpoint)
showrun echo "Entering shared endpoint setup"
# Use existing host bits when testing is to happen inside a container
# since this script will run again in that environment.
# shellcheck disable=SC2154
@ -397,21 +411,21 @@ case "$TEST_FLAVOR" in
install_test_configs
;;
minikube)
dnf install -y $PACKAGE_DOWNLOAD_DIR/minikube-latest*
showrun dnf install -y $PACKAGE_DOWNLOAD_DIR/minikube-latest*
remove_packaged_podman_files
make install.tools
make install PREFIX=/usr ETCDIR=/etc
minikube config set driver podman
showrun make install.tools
showrun make install PREFIX=/usr ETCDIR=/etc
showrun minikube config set driver podman
install_test_configs
;;
machine)
dnf install -y podman-gvproxy*
showrun dnf install -y podman-gvproxy*
remove_packaged_podman_files
make install PREFIX=/usr ETCDIR=/etc
showrun make install PREFIX=/usr ETCDIR=/etc
install_test_configs
;;
swagger)
make .install.swagger
showrun make .install.swagger
;;
#fcos_image_build)
# ;;
@ -425,6 +439,7 @@ esac
if [[ ! "$OS_RELEASE_ID" =~ "debian" ]] && \
[[ "$CIRRUS_CHANGE_TITLE" =~ CI:NEXT ]]
then
showrun echo "Entering setup for CI:NEXT"
# shellcheck disable=SC2154
if [[ "$CIRRUS_PR_DRAFT" != "true" ]]; then
die "Magic 'CI:NEXT' string can only be used on DRAFT PRs"
@ -448,3 +463,5 @@ echo -e "\n# End of global variable definitions" \
msg "Global CI Environment vars.:"
grep -Ev '^#' /etc/ci_environment | sort | indent
showrun echo "finished"