opensource/podman
1
0
Fork 0
mirror of https://github.com/containers/podman.git synced 2025-05-20 08:36:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

Cirrus: Unify package installation

Browse Source 
Also, test-build critical container images depended upon for
CI-purposes.

Signed-off-by: Chris Evich <cevich@redhat.com>
This commit is contained in:
Chris Evich
2020-04-15 12:01:23 -04:00
parent 2c457a632c
commit 97ecd21b59
12 changed files with 463 additions and 577 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
.cirrus.yml
View File

@ -156,6 +156,32 @@ gating_task:
failed_branch_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_branch_failure.sh'
# Ensure these container images can build
container_image_build_task:
alias: 'container_image_build'
depends_on:
- "gating"
# Only run for PRs, quay.io will automatically build after bramch-push
only_if: $CIRRUS_BRANCH != $DEST_BRANCH
matrix:
- name: "build in_podman image ${FEDORA_NAME} "
container:
dockerfile: Dockerfile
- name: "build in_podman image ${UBUNTU_NAME}"
container:
dockerfile: Dockerfile.ubuntu
- name: "build gate image $DEST_BRANCH branch"
container:
dockerfile: contrib/gate/Dockerfile
container:
dockerfile: Dockerfile
script: make install.remote
# This task checks to make sure that we can still build an rpm from the
# source code using contrib/rpm/podman.spec.in
rpmbuild_task:
@ -389,6 +415,7 @@ testing_task:
- "varlink_api"
- "build_each_commit"
- "build_without_cgo"
- "container_image_build"
allow_failures: $CI == 'true'
@ -681,6 +708,7 @@ test_build_cache_images_task:
depends_on:
- "gating"
- 'container_image_build'
# VMs created by packer are not cleaned up by cirrus, must allow task to complete
auto_cancellation: $CI != "true"
@ -782,6 +810,7 @@ success_task:
- "varlink_api"
- "build_each_commit"
- "build_without_cgo"
- "container_image_build"
- "meta"
- "image_prune"
- "testing"

135
Dockerfile
View File

@ -1,117 +1,26 @@
FROM golang:1.12
FROM registry.fedoraproject.org/fedora:latest
RUN apt-get update && apt-get install -y \
apparmor \
autoconf \
automake \
bison \
build-essential \
curl \
e2fslibs-dev \
file \
gawk \
gettext \
go-md2man \
iptables \
pkg-config \
libaio-dev \
libcap-dev \
libfuse-dev \
libnet-dev \
libnl-3-dev \
libprotobuf-dev \
libprotobuf-c-dev \
libseccomp2 \
libseccomp-dev \
libtool \
libudev-dev \
protobuf-c-compiler \
protobuf-compiler \
libglib2.0-dev \
libapparmor-dev \
btrfs-tools \
libdevmapper1.02.1 \
libdevmapper-dev \
libgpgme11-dev \
liblzma-dev \
netcat \
socat \
lsof \
xz-utils \
unzip \
python3-yaml \
--no-install-recommends \
&& apt-get clean
# This container image is utilized by the containers CI automation system
# for building and testing libpod inside a container environment.
# It is assumed that the source to be tested will overwrite $GOSRC (below)
# at runtime.
ENV GOPATH=/var/tmp/go
ENV GOSRC=$GOPATH/src/github.com/containers/libpod
ENV SCRIPT_BASE=./contrib/cirrus
ENV PACKER_BASE=$SCRIPT_BASE/packer
# Install runc
ENV RUNC_COMMIT 029124da7af7360afa781a0234d1b083550f797c
RUN set -x \
&& export GOPATH="$(mktemp -d)" \
&& git clone https://github.com/opencontainers/runc.git "$GOPATH/src/github.com/opencontainers/runc" \
&& cd "$GOPATH/src/github.com/opencontainers/runc" \
&& git fetch origin --tags \
&& git checkout --detach -q "$RUNC_COMMIT" \
&& make static BUILDTAGS="seccomp selinux" \
&& cp runc /usr/bin/runc \
&& rm -rf "$GOPATH"
# Only add minimal tooling necessary to complete setup.
ADD /$SCRIPT_BASE $GOSRC/$SCRIPT_BASE
ADD /hack/install_catatonit.sh $GOSRC/hack/
ADD /cni/*.conflist $GOSRC/cni/
ADD /test/*.json $GOSRC/test/
ADD /test/*.conf $GOSRC/test/
WORKDIR $GOSRC
# Install conmon
ENV CONMON_COMMIT 65fe0226d85b69fc9e527e376795c9791199153d
RUN set -x \
&& export GOPATH="$(mktemp -d)" \
&& git clone https://github.com/containers/conmon.git "$GOPATH/src/github.com/containers/conmon.git" \
&& cd "$GOPATH/src/github.com/containers/conmon.git" \
&& git fetch origin --tags \
&& git checkout --detach -q "$CONMON_COMMIT" \
&& make \
&& install -D -m 755 bin/conmon /usr/libexec/podman/conmon \
&& rm -rf "$GOPATH"
# Re-use repositories and package setup as in VMs under CI
RUN bash $PACKER_BASE/fedora_packaging.sh && \
dnf clean all && \
rm -rf /var/cache/dnf
# Install CNI plugins
ENV CNI_COMMIT 485be65581341430f9106a194a98f0f2412245fb
RUN set -x \
&& export GOPATH="$(mktemp -d)" GOCACHE="$(mktemp -d)" \
&& git clone https://github.com/containernetworking/plugins.git "$GOPATH/src/github.com/containernetworking/plugins" \
&& cd "$GOPATH/src/github.com/containernetworking/plugins" \
&& git checkout --detach -q "$CNI_COMMIT" \
&& ./build_linux.sh \
&& mkdir -p /usr/libexec/cni \
&& cp bin/* /usr/libexec/cni \
&& rm -rf "$GOPATH"
# Install ginkgo
RUN set -x \
&& export GOPATH=/go \
&& go get -u github.com/onsi/ginkgo/ginkgo \
&& install -D -m 755 "$GOPATH"/bin/ginkgo /usr/bin/
# Install gomega
RUN set -x \
&& export GOPATH=/go \
&& go get github.com/onsi/gomega/...
# Install latest stable criu version
RUN set -x \
&& cd /tmp \
&& git clone https://github.com/checkpoint-restore/criu.git \
&& cd criu \
&& make \
&& install -D -m 755 criu/criu /usr/sbin/ \
&& rm -rf /tmp/criu
# Install cni config
#RUN make install.cni
RUN mkdir -p /etc/cni/net.d/
COPY cni/87-podman-bridge.conflist /etc/cni/net.d/87-podman-bridge.conflist
# Make sure we have some policy for pulling images
RUN mkdir -p /etc/containers && curl https://raw.githubusercontent.com/projectatomic/registries/master/registries.fedora -o /etc/containers/registries.conf
COPY test/policy.json /etc/containers/policy.json
COPY test/redhat_sigstore.yaml /etc/containers/registries.d/registry.access.redhat.com.yaml
ADD . /go/src/github.com/containers/libpod
RUN set -x && cd /go/src/github.com/containers/libpod
WORKDIR /go/src/github.com/containers/libpod
# Mirror steps taken under CI
RUN bash -c 'source $GOSRC/$SCRIPT_BASE/lib.sh && install_test_configs'

77
Dockerfile.centos
View File

@ -1,77 +0,0 @@
FROM registry.centos.org/centos/centos:7
RUN yum -y install btrfs-progs-devel \
atomic-registries \
autoconf \
automake \
bzip2 \
device-mapper-devel \
findutils \
file \
git \
glibc-static \
glib2-devel \
gnupg \
golang \
golang-github-cpuguy83-go-md2man \
gpgme-devel \
libassuan-devel \
libseccomp-devel \
libselinux-devel \
libtool \
containers-common \
runc \
make \
lsof \
which\
golang-github-cpuguy83-go-md2man \
nmap-ncat \
xz \
iptables && yum clean all
# Install CNI plugins
ENV CNI_COMMIT 485be65581341430f9106a194a98f0f2412245fb
RUN set -x \
&& export GOPATH="$(mktemp -d)" GOCACHE="$(mktemp -d)" \
&& git clone https://github.com/containernetworking/plugins.git "$GOPATH/src/github.com/containernetworking/plugins" \
&& cd "$GOPATH/src/github.com/containernetworking/plugins" \
&& git checkout --detach -q "$CNI_COMMIT" \
&& ./build_linux.sh \
&& mkdir -p /usr/libexec/cni \
&& cp bin/* /usr/libexec/cni \
&& rm -rf "$GOPATH"
# Install ginkgo
RUN set -x \
&& export GOPATH=/go \
&& go get -u github.com/onsi/ginkgo/ginkgo \
&& install -D -m 755 "$GOPATH"/bin/ginkgo /usr/bin/
# Install gomega
RUN set -x \
&& export GOPATH=/go \
&& go get github.com/onsi/gomega/...
# Install conmon
ENV CONMON_COMMIT 6f3572558b97bc60dd8f8c7f0807748e6ce2c440
RUN set -x \
&& export GOPATH="$(mktemp -d)" \
&& git clone https://github.com/containers/conmon.git "$GOPATH/src/github.com/containers/conmon.git" \
&& cd "$GOPATH/src/github.com/containers/conmon.git" \
&& git fetch origin --tags \
&& git checkout --detach -q "$CONMON_COMMIT" \
&& make \
&& install -D -m 755 bin/conmon /usr/libexec/podman/conmon \
&& rm -rf "$GOPATH"
# Install cni config
#RUN make install.cni
RUN mkdir -p /etc/cni/net.d/
COPY cni/87-podman-bridge.conflist /etc/cni/net.d/87-podman-bridge.conflist
# Make sure we have some policy for pulling images
RUN mkdir -p /etc/containers
COPY test/policy.json /etc/containers/policy.json
COPY test/redhat_sigstore.yaml /etc/containers/registries.d/registry.access.redhat.com.yaml
WORKDIR /go/src/github.com/containers/libpod

73
Dockerfile.fedora
View File

@ -1,73 +0,0 @@
FROM registry.fedoraproject.org/fedora:30
RUN dnf -y install btrfs-progs-devel \
atomic-registries \
autoconf \
automake \
bzip2 \
device-mapper-devel \
file \
findutils \
git \
glib2-devel \
glibc-static \
gnupg \
golang \
golang-github-cpuguy83-go-md2man \
gpgme-devel \
libassuan-devel \
libseccomp-devel \
libselinux-devel \
libtool \
containers-common \
runc \
make \
lsof \
which\
golang-github-cpuguy83-go-md2man \
procps-ng \
nmap-ncat \
xz \
slirp4netns \
container-selinux \
containernetworking-plugins \
iproute \
iptables && dnf clean all
# Install ginkgo
RUN set -x \
&& export GOPATH=/go GOCACHE="$(mktemp -d)" \
&& go get -u github.com/onsi/ginkgo/ginkgo \
&& install -D -m 755 "$GOPATH"/bin/ginkgo /usr/bin/
# Install gomega
RUN set -x \
&& export GOPATH=/go GOCACHE="$(mktemp -d)" \
&& go get github.com/onsi/gomega/...
# Install conmon
ENV CONMON_COMMIT 6f3572558b97bc60dd8f8c7f0807748e6ce2c440
RUN set -x \
&& export GOPATH="$(mktemp -d)" GOCACHE="$(mktemp -d)" \
&& git clone https://github.com/containers/conmon.git "$GOPATH/src/github.com/containers/conmon.git" \
&& cd "$GOPATH/src/github.com/containers/conmon.git" \
&& git fetch origin --tags \
&& git checkout --detach -q "$CONMON_COMMIT" \
&& make \
&& install -D -m 755 bin/conmon /usr/libexec/podman/conmon \
&& rm -rf "$GOPATH"
# Install cni config
#RUN make install.cni
RUN mkdir -p /etc/cni/net.d/
COPY cni/87-podman-bridge.conflist /etc/cni/net.d/87-podman-bridge.conflist
# Make sure we have some policy for pulling images
RUN mkdir -p /etc/containers
COPY test/policy.json /etc/containers/policy.json
COPY test/redhat_sigstore.yaml /etc/containers/registries.d/registry.access.redhat.com.yaml
# Install varlink stuff
RUN pip3 install varlink
WORKDIR /go/src/github.com/containers/libpod

29
Dockerfile.ubuntu Normal file
View File

@ -0,0 +1,29 @@
# Must resemble $UBUNTU_BASE_IMAGE in ./contrib/cirrus/lib.sh
FROM ubuntu:latest
# This container image is intended for building and testing libpod
# from inside a container environment. It is assumed that the source
# to be tested will overwrite $GOSRC (below) at runtime.
ENV GOPATH=/var/tmp/go
ENV GOSRC=$GOPATH/src/github.com/containers/libpod
ENV SCRIPT_BASE=./contrib/cirrus
ENV PACKER_BASE=$SCRIPT_BASE/packer
RUN export DEBIAN_FRONTEND="noninteractive" && \
apt-get -qq update --yes && \
apt-get -qq upgrade --yes && \
apt-get -qq install curl git && \
apt-get -qq autoremove --yes && \
rm -rf /var/cache/apt
# Only add minimal tooling necessary to complete setup.
ADD / $GOSRC
WORKDIR $GOSRC
# Re-use repositories and package setup as in VMs under CI
RUN bash $PACKER_BASE/ubuntu_packaging.sh && \
apt-get -qq autoremove --yes && \
rm -rf /var/cache/apt
# Mirror steps taken under CI
RUN bash -c 'source $GOSRC/$SCRIPT_BASE/lib.sh && install_test_configs'

83
contrib/cirrus/lib.sh
View File

@ -6,6 +6,11 @@
# Global details persist here
source /etc/environment # not always loaded under all circumstances
# Automation environment doesn't automatically load for Ubuntu 18
if [[ -r '/usr/share/automation/environment' ]]; then
source '/usr/share/automation/environment'
fi
# Under some contexts these values are not set, make sure they are.
export USER="$(whoami)"
export HOME="$(getent passwd $USER | cut -d : -f 6)"
@ -72,10 +77,15 @@ IN_PODMAN_IMAGE="quay.io/libpod/in_podman:$DEST_BRANCH"
# Image for uploading releases
UPLDREL_IMAGE="quay.io/libpod/upldrel:master"
# This is needed under some environments/contexts
SUDO=''
[[ "$UID" -eq 0 ]] || \
SUDO='sudo -E'
# Avoid getting stuck waiting for user input
export DEBIAN_FRONTEND="noninteractive"
SUDOAPTGET="ooe.sh sudo -E apt-get -qq --yes"
SUDOAPTADD="ooe.sh sudo -E add-apt-repository --yes"
SUDOAPTGET="$SUDO apt-get -qq --yes"
SUDOAPTADD="$SUDO add-apt-repository --yes"
# Regex that finds enabled periodic apt configuration items
PERIODIC_APT_RE='^(APT::Periodic::.+")1"\;'
# Short-cuts for retrying/timeout calls
@ -109,6 +119,9 @@ OS_REL_VER="${OS_RELEASE_ID}-${OS_RELEASE_VER}"
# Type of filesystem used for cgroups
CG_FS_TYPE="$(stat -f -c %T /sys/fs/cgroup)"
# When building images, the version of automation tooling to install
INSTALL_AUTOMATION_VERSION=1.1.3
# Installed into cache-images, supports overrides
# by user-data in case of breakage or for debugging.
CUSTOM_CLOUD_CONFIG_DEFAULTS="$GOSRC/$PACKER_BASE/cloud-init/$OS_RELEASE_ID/cloud.cfg.d"
@ -354,25 +367,18 @@ setup_rootless() {
die 11 "Timeout exceeded waiting for localhost ssh capability"
}
# Helper/wrapper script to only show stderr/stdout on non-zero exit
install_ooe() {
req_env_var SCRIPT_BASE
echo "Installing script to mask stdout/stderr unless non-zero exit."
sudo install -D -m 755 "$GOSRC/$SCRIPT_BASE/ooe.sh" /usr/local/bin/ooe.sh
}
# Grab a newer version of git from software collections
# https://www.softwarecollections.org/en/
# and use it with a wrapper
install_scl_git() {
echo "Installing SoftwareCollections updated 'git' version."
ooe.sh sudo yum -y install rh-git29
cat << "EOF" | sudo tee /usr/bin/git
ooe.sh $SUDO yum -y install rh-git29
cat << "EOF" | $SUDO tee /usr/bin/git
#!/bin/bash
scl enable rh-git29 -- git $@
EOF
sudo chmod 755 /usr/bin/git
$SUDO chmod 755 /usr/bin/git
}
install_test_configs() {
@ -414,9 +420,9 @@ remove_packaged_podman_files() {
if [[ "$OS_RELEASE_ID" =~ "ubuntu" ]]
then
LISTING_CMD="sudo -E dpkg-query -L podman"
LISTING_CMD="$SUDO dpkg-query -L podman"
else
LISTING_CMD='sudo rpm -ql podman'
LISTING_CMD='$SUDO rpm -ql podman'
fi
# yum/dnf/dpkg may list system directories, only remove files
@ -424,7 +430,7 @@ remove_packaged_podman_files() {
do
# Sub-directories may contain unrelated/valuable stuff
if [[ -d "$fullpath" ]]; then continue; fi
ooe.sh sudo rm -vf "$fullpath"
ooe.sh $SUDO rm -vf "$fullpath"
done
# Be super extra sure and careful vs performant and completely safe
@ -447,43 +453,60 @@ systemd_banish() {
$GOSRC/$PACKER_BASE/systemd_banish.sh
}
# This can be removed when the kernel bug fix is included in Fedora
workaround_bfq_bug() {
if [[ "$OS_RELEASE_ID" == "fedora" ]] && [[ $OS_RELEASE_VER -le 32 ]]; then
warn "Switching io scheduler to 'deadline' to avoid RHBZ 1767539"
warn "aka https://bugzilla.kernel.org/show_bug.cgi?id=205447"
echo "mq-deadline" | sudo tee /sys/block/sda/queue/scheduler > /dev/null
echo -n "IO Scheduler set to: "
$SUDO cat /sys/block/sda/queue/scheduler
fi
}
# Warning: DO NOT USE.
# This is called by other functions as the very last step during the VM Image build
# process. It's purpose is to "reset" the image, so all the first-boot operations
# happen at test runtime (like generating new ssh host keys, resizing partitions, etc.)
_finalize() {
set +e # Don't fail at the very end
if [[ -d "$CUSTOM_CLOUD_CONFIG_DEFAULTS" ]]
then
echo "Installing custom cloud-init defaults"
sudo cp -v "$CUSTOM_CLOUD_CONFIG_DEFAULTS"/* /etc/cloud/cloud.cfg.d/
$SUDO cp -v "$CUSTOM_CLOUD_CONFIG_DEFAULTS"/* /etc/cloud/cloud.cfg.d/
else
echo "Could not find any files in $CUSTOM_CLOUD_CONFIG_DEFAULTS"
fi
echo "Re-initializing so next boot does 'first-boot' setup again."
cd /
sudo rm -rf /var/lib/cloud/instanc*
sudo rm -rf /root/.ssh/*
sudo rm -rf /etc/ssh/*key*
sudo rm -rf /etc/ssh/moduli
sudo rm -rf /home/*
sudo rm -rf /tmp/*
sudo rm -rf /tmp/.??*
sudo sync
sudo fstrim -av
$SUDO rm -rf /var/lib/cloud/instanc*
$SUDO rm -rf /root/.ssh/*
$SUDO rm -rf /etc/ssh/*key*
$SUDO rm -rf /etc/ssh/moduli
$SUDO rm -rf /home/*
$SUDO rm -rf /tmp/*
$SUDO rm -rf /tmp/.??*
$SUDO sync
$SUDO fstrim -av
}
# Called during VM Image setup, not intended for general use.
rh_finalize() {
set +e # Don't fail at the very end
echo "Resetting to fresh-state for usage as cloud-image."
PKG=$(type -P dnf || type -P yum || echo "")
sudo $PKG clean all
sudo rm -rf /var/cache/{yum,dnf}
sudo rm -f /etc/udev/rules.d/*-persistent-*.rules
sudo touch /.unconfigured # force firstboot to run
$SUDO $PKG clean all
$SUDO rm -rf /var/cache/{yum,dnf}
$SUDO rm -f /etc/udev/rules.d/*-persistent-*.rules
$SUDO touch /.unconfigured # force firstboot to run
_finalize
}
# Called during VM Image setup, not intended for general use.
ubuntu_finalize() {
set +e # Don't fail at the very end
echo "Resetting to fresh-state for usage as cloud-image."
$LILTO $SUDOAPTGET autoremove
sudo rm -rf /var/cache/apt
$SUDO rm -rf /var/cache/apt
_finalize
}

141
contrib/cirrus/packer/fedora_packaging.sh Normal file
View File

@ -0,0 +1,141 @@
#!/bin/bash
# This script is called from fedora_setup.sh and various Dockerfiles.
# It's not intended to be used outside of those contexts. It assumes the lib.sh
# library has already been sourced, and that all "ground-up" package-related activity
# needs to be done, including repository setup and initial update.
set -e
echo "Updating/Installing repos and packages for $OS_REL_VER"
source $GOSRC/$SCRIPT_BASE/lib.sh
# Pre-req. to install automation tooing
$LILTO $SUDO dnf install -y git
# Install common automation tooling (i.e. ooe.sh)
curl --silent --show-error --location \
--url "https://raw.githubusercontent.com/containers/automation/master/bin/install_automation.sh" | \
$SUDO env INSTALL_PREFIX=/usr/share /bin/bash -s - "$INSTALL_AUTOMATION_VERSION"
# Reload installed environment right now (happens automatically in a new process)
source /usr/share/automation/environment
# Set this to 1 to NOT enable updates-testing repository
DISABLE_UPDATES_TESTING=${DISABLE_UPDATES_TESTING:0}
# Do not enable update-stesting on the previous Fedora release
if ((DISABLE_UPDATES_TESTING!=0)); then
warn "Enabling updates-testing repository for image based on $FEDORA_BASE_IMAGE"
$LILTO $SUDO ooe.sh dnf install -y 'dnf-command(config-manager)'
$LILTO $SUDO ooe.sh dnf config-manager --set-enabled updates-testing
else
warn "NOT enabling updates-testing repository for image based on $PRIOR_FEDORA_BASE_IMAGE"
fi
$BIGTO ooe.sh $SUDO dnf update -y
REMOVE_PACKAGES=()
INSTALL_PACKAGES=(\
autoconf
automake
bash-completion
bats
bridge-utils
btrfs-progs-devel
buildah
bzip2
conmon
container-selinux
containernetworking-plugins
containers-common
criu
device-mapper-devel
dnsmasq
emacs-nox
file
findutils
fuse3
fuse3-devel
gcc
git
glib2-devel
glibc-static
gnupg
go-md2man
golang
gpgme-devel
iproute
iptables
jq
libassuan-devel
libcap-devel
libmsi1
libnet
libnet-devel
libnl3-devel
libseccomp
libseccomp-devel
libselinux-devel
libtool
libvarlink-util
lsof
make
msitools
nmap-ncat
ostree-devel
pandoc
podman
procps-ng
protobuf
protobuf-c
protobuf-c-devel
protobuf-devel
python
python3-dateutil
python3-psutil
python3-pytoml
rsync
selinux-policy-devel
skopeo
skopeo-containers
slirp4netns
unzip
vim
wget
which
xz
zip
)
case "$OS_RELEASE_VER" in
30)
INSTALL_PACKAGES+=(\
atomic-registries
golang-github-cpuguy83-go-md2man
python2-future
runc
)
REMOVE_PACKAGES+=(crun)
;;
31)
INSTALL_PACKAGES+=(crun)
REMOVE_PACKAGES+=(runc)
;;
32)
INSTALL_PACKAGES+=(crun)
REMOVE_PACKAGES+=(runc)
;;
*)
bad_os_id_ver ;;
esac
echo "Installing general build/test dependencies for Fedora '$OS_RELEASE_VER'"
$BIGTO ooe.sh $SUDO dnf install -y ${INSTALL_PACKAGES[@]}
[[ ${#REMOVE_PACKAGES[@]} -eq 0 ]] || \
$LILTO ooe.sh $SUDO dnf erase -y ${REMOVE_PACKAGES[@]}
export GOPATH="$(mktemp -d)"
trap "$SUDO rm -rf $GOPATH" EXIT
ooe.sh $SUDO $GOSRC/hack/install_catatonit.sh

129
contrib/cirrus/packer/fedora_setup.sh
View File

@ -6,139 +6,26 @@
set -e
# Load in library (copied by packer, before this script was run)
source /tmp/libpod/$SCRIPT_BASE/lib.sh
source $GOSRC/$SCRIPT_BASE/lib.sh
req_env_var SCRIPT_BASE PACKER_BUILDER_NAME GOSRC FEDORA_BASE_IMAGE OS_RELEASE_ID OS_RELEASE_VER
req_env_var SCRIPT_BASE PACKER_BASE INSTALL_AUTOMATION_VERSION PACKER_BUILDER_NAME GOSRC FEDORA_BASE_IMAGE OS_RELEASE_ID OS_RELEASE_VER
install_ooe
if [[ $OS_RELEASE_VER -le 31 ]]; then
warn "Switching io scheduler to 'deadline' to avoid RHBZ 1767539"
warn "aka https://bugzilla.kernel.org/show_bug.cgi?id=205447"
echo "mq-deadline" | sudo tee /sys/block/sda/queue/scheduler > /dev/null
sudo cat /sys/block/sda/queue/scheduler
fi
export GOPATH="$(mktemp -d)"
trap "sudo rm -rf $GOPATH" EXIT
$BIGTO ooe.sh sudo dnf update -y
workaround_bfq_bug
# Do not enable update-stesting on the previous Fedora release
if [[ "$FEDORA_BASE_IMAGE" =~ "${OS_RELEASE_ID}-cloud-base-${OS_RELEASE_VER}" ]]; then
warn "Enabling updates-testing repository for image based on $FEDORA_BASE_IMAGE"
$LILTO ooe.sh sudo dnf install -y 'dnf-command(config-manager)'
$LILTO ooe.sh sudo dnf config-manager --set-enabled updates-testing
DISABLE_UPDATES_TESTING=0
else
warn "NOT enabling updates-testing repository for image based on $PRIOR_FEDORA_BASE_IMAGE"
DISABLE_UPDATES_TESTING=1
fi
REMOVE_PACKAGES=()
INSTALL_PACKAGES=(\
autoconf
automake
bash-completion
bats
bridge-utils
btrfs-progs-devel
buildah
bzip2
conmon
container-selinux
containernetworking-plugins
containers-common
criu
device-mapper-devel
dnsmasq
emacs-nox
file
findutils
fuse3
fuse3-devel
gcc
git
glib2-devel
glibc-static
gnupg
go-md2man
golang
gpgme-devel
iproute
iptables
jq
libassuan-devel
libcap-devel
libmsi1
libnet
libnet-devel
libnl3-devel
libseccomp
libseccomp-devel
libselinux-devel
libtool
libvarlink-util
lsof
make
msitools
nmap-ncat
ostree-devel
pandoc
podman
procps-ng
protobuf
protobuf-c
protobuf-c-devel
protobuf-devel
python
python3-dateutil
python3-psutil
python3-pytoml
rsync
selinux-policy-devel
skopeo
skopeo-containers
slirp4netns
unzip
vim
wget
which
xz
zip
)
case "$OS_RELEASE_VER" in
30)
INSTALL_PACKAGES+=(\
atomic-registries
golang-github-cpuguy83-go-md2man
python2-future
runc
)
REMOVE_PACKAGES+=(crun)
;;
31)
INSTALL_PACKAGES+=(crun)
REMOVE_PACKAGES+=(runc)
;;
32)
INSTALL_PACKAGES+=(crun)
REMOVE_PACKAGES+=(runc)
;;
*)
bad_os_id_ver ;;
esac
echo "Installing general build/test dependencies for Fedora '$OS_RELEASE_VER'"
$BIGTO ooe.sh sudo dnf install -y ${INSTALL_PACKAGES[@]}
[[ "${#REMOVE_PACKAGES[@]}" -eq "0" ]] || \
$LILTO ooe.sh sudo dnf erase -y ${REMOVE_PACKAGES[@]}
bash $PACKER_BASE/fedora_packaging.sh
# Load installed environment right now (happens automatically in a new process)
source /usr/share/automation/environment
echo "Enabling cgroup management from containers"
ooe.sh sudo setsebool container_manage_cgroup true
ooe.sh sudo /tmp/libpod/hack/install_catatonit.sh
# Ensure there are no disruptive periodic services enabled by default in image
systemd_banish

1
contrib/cirrus/packer/libpod_images.yml
View File

@ -71,6 +71,7 @@ provisioners:
environment_vars:
- 'PACKER_BUILDER_NAME={{build_name}}'
- 'GOSRC=/tmp/libpod'
- 'PACKER_BASE={{user `PACKER_BASE`}}'
- 'SCRIPT_BASE={{user `SCRIPT_BASE`}}'
post-processors:

168
contrib/cirrus/packer/ubuntu_packaging.sh Normal file
View File

@ -0,0 +1,168 @@
#!/bin/bash
# This script is called from ubuntu_setup.sh and various Dockerfiles.
# It's not intended to be used outside of those contexts. It assumes the lib.sh
# library has already been sourced, and that all "ground-up" package-related activity
# needs to be done, including repository setup and initial update.
set -e
echo "Updating/Installing repos and packages for $OS_REL_VER"
source $GOSRC/$SCRIPT_BASE/lib.sh
echo "Updating/configuring package repositories."
$BIGTO $SUDOAPTGET update
echo "Installing deps to add third-party repositories and automation tooling"
$LILTO $SUDOAPTGET install software-properties-common git curl
# Install common automation tooling (i.e. ooe.sh)
curl --silent --show-error --location \
--url "https://raw.githubusercontent.com/containers/automation/master/bin/install_automation.sh" | \
$SUDO env INSTALL_PREFIX=/usr/share /bin/bash -s - "$INSTALL_AUTOMATION_VERSION"
# Reload installed environment right now (happens automatically in a new process)
source /usr/share/automation/environment
$LILTO ooe.sh $SUDOAPTADD ppa:criu/ppa
# Install newer version of golang
if [[ "$OS_RELEASE_VER" -eq "18" ]]
then
$LILTO ooe.sh $SUDOAPTADD ppa:longsleep/golang-backports
fi
echo "Configuring/Instaling deps from Open build server"
VERSION_ID=$(source /etc/os-release; echo $VERSION_ID)
echo "deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_$VERSION_ID/ /" \
| ooe.sh $SUDO tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list
ooe.sh curl -L -o /tmp/Release.key "https://download.opensuse.org/repositories/devel:kubic:libcontainers:stable/xUbuntu_${VERSION_ID}/Release.key"
ooe.sh $SUDO apt-key add - < /tmp/Release.key
INSTALL_PACKAGES=(\
apparmor
aufs-tools
autoconf
automake
bash-completion
bison
build-essential
buildah
bzip2
conmon
containernetworking-plugins
containers-common
coreutils
cri-o-runc
criu
curl
dnsmasq
e2fslibs-dev
emacs-nox
file
gawk
gcc
gettext
git
go-md2man
golang
iproute2
iptables
jq
libaio-dev
libapparmor-dev
libcap-dev
libdevmapper-dev
libdevmapper1.02.1
libfuse-dev
libfuse2
libglib2.0-dev
libgpgme11-dev
liblzma-dev
libnet1
libnet1-dev
libnl-3-dev
libprotobuf-c-dev
libprotobuf-dev
libseccomp-dev
libseccomp2
libselinux-dev
libsystemd-dev
libtool
libudev-dev
libvarlink
lsof
make
netcat
openssl
pkg-config
podman
protobuf-c-compiler
protobuf-compiler
python-future
python-minimal
python-protobuf
python3-dateutil
python3-pip
python3-psutil
python3-pytoml
python3-setuptools
rsync
runc
scons
skopeo
slirp4netns
socat
sudo
unzip
vim
wget
xz-utils
yum-utils
zip
zlib1g-dev
)
if [[ $OS_RELEASE_VER -ge 19 ]]
then
INSTALL_PACKAGES+=(\
bats
btrfs-progs
fuse3
libbtrfs-dev
libfuse3-dev
)
else
echo "Downloading version of bats with fix for a \$IFS related bug in 'run' command"
cd /tmp
BATS_URL='http://launchpadlibrarian.net/438140887/bats_1.1.0+git104-g1c83a1b-1_all.deb'
curl -L -O "$BATS_URL"
cd -
INSTALL_PACKAGES+=(\
/tmp/$(basename $BATS_URL)
btrfs-tools
)
fi
# Do this at the last possible moment to avoid dpkg lock conflicts
echo "Upgrading all packages"
$BIGTO ooe.sh $SUDOAPTGET upgrade
echo "Installing general testing and system dependencies"
# Necessary to update cache of newly added repos
$LILTO ooe.sh $SUDOAPTGET update
$BIGTO ooe.sh $SUDOAPTGET install ${INSTALL_PACKAGES[@]}
export GOPATH="$(mktemp -d)"
trap "$SUDO rm -rf $GOPATH" EXIT
echo "Installing cataonit and libseccomp.sudo"
cd $GOSRC
ooe.sh $SUDO hack/install_catatonit.sh
ooe.sh $SUDO make install.libseccomp.sudo
CRIO_RUNC_PATH="/usr/lib/cri-o-runc/sbin/runc"
if $SUDO dpkg -L cri-o-runc | grep -m 1 -q "$CRIO_RUNC_PATH"
then
echo "Linking $CRIO_RUNC_PATH to /usr/bin/runc for ease of testing."
$SUDO ln -f "$CRIO_RUNC_PATH" "/usr/bin/runc"
fi

165
contrib/cirrus/packer/ubuntu_setup.sh
View File

@ -8,164 +8,21 @@ set -e
# Load in library (copied by packer, before this script was run)
source $GOSRC/$SCRIPT_BASE/lib.sh
req_env_var SCRIPT_BASE
install_ooe
export GOPATH="$(mktemp -d)"
trap "sudo rm -rf $GOPATH" EXIT
# Stop disruption upon boot ASAP after booting
echo "Disabling all packaging activity on boot"
# Don't let sed process sed's temporary files
_FILEPATHS=$(sudo ls -1 /etc/apt/apt.conf.d)
for filename in $_FILEPATHS; do \
echo "Checking/Patching $filename"
sudo sed -i -r -e "s/$PERIODIC_APT_RE/"'\10"\;/' "/etc/apt/apt.conf.d/$filename"; done
echo "Updating/configuring package repositories."
$BIGTO $SUDOAPTGET update
echo "Upgrading all packages"
$BIGTO $SUDOAPTGET upgrade
echo "Adding third-party repositories and PPAs"
$LILTO $SUDOAPTGET install software-properties-common
$LILTO $SUDOAPTADD ppa:criu/ppa
if [[ "$OS_RELEASE_VER" -eq "18" ]]
then
$LILTO $SUDOAPTADD ppa:longsleep/golang-backports
fi
echo "Configuring/Instaling deps from Open build server"
VERSION_ID=$(source /etc/os-release; echo $VERSION_ID)
echo "deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_$VERSION_ID/ /" \
| ooe.sh sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list
ooe.sh curl -L -o /tmp/Release.key "https://download.opensuse.org/repositories/devel:kubic:libcontainers:stable/xUbuntu_${VERSION_ID}/Release.key"
ooe.sh sudo apt-key add - < /tmp/Release.key
INSTALL_PACKAGES=(\
apparmor
aufs-tools
autoconf
automake
bash-completion
bison
build-essential
buildah
bzip2
conmon
containernetworking-plugins
containers-common
coreutils
cri-o-runc
criu
curl
dnsmasq
e2fslibs-dev
emacs-nox
file
gawk
gcc
gettext
git
go-md2man
golang
iproute2
iptables
jq
libaio-dev
libapparmor-dev
libcap-dev
libdevmapper-dev
libdevmapper1.02.1
libfuse-dev
libfuse2
libglib2.0-dev
libgpgme11-dev
liblzma-dev
libnet1
libnet1-dev
libnl-3-dev
libprotobuf-c-dev
libprotobuf-dev
libseccomp-dev
libseccomp2
libselinux-dev
libsystemd-dev
libtool
libudev-dev
libvarlink
lsof
make
netcat
openssl
pkg-config
podman
protobuf-c-compiler
protobuf-compiler
python-future
python-minimal
python-protobuf
python3-dateutil
python3-pip
python3-psutil
python3-pytoml
python3-setuptools
rsync
runc
scons
skopeo
slirp4netns
socat
unzip
vim
wget
xz-utils
yum-utils
zip
zlib1g-dev
)
if [[ "$OS_RELEASE_VER" -ge "19" ]]
then
INSTALL_PACKAGES+=(\
bats
btrfs-progs
fuse3
libbtrfs-dev
libfuse3-dev
)
else
echo "Downloading version of bats with fix for a \$IFS related bug in 'run' command"
cd /tmp
BATS_URL='http://launchpadlibrarian.net/438140887/bats_1.1.0+git104-g1c83a1b-1_all.deb'
curl -L -O "$BATS_URL"
cd -
INSTALL_PACKAGES+=(\
/tmp/$(basename $BATS_URL)
btrfs-tools
)
fi
echo "Installing general testing and system dependencies"
# Necessary to update cache of newly added repos
$LILTO $SUDOAPTGET update
$BIGTO $SUDOAPTGET install ${INSTALL_PACKAGES[@]}
echo "Installing cataonit and libseccomp.sudo"
ooe.sh sudo /tmp/libpod/hack/install_catatonit.sh
ooe.sh sudo make -C /tmp/libpod install.libseccomp.sudo
req_env_var SCRIPT_BASE PACKER_BASE INSTALL_AUTOMATION_VERSION PACKER_BUILDER_NAME GOSRC UBUNTU_BASE_IMAGE OS_RELEASE_ID OS_RELEASE_VER
# Ensure there are no disruptive periodic services enabled by default in image
systemd_banish
CRIO_RUNC_PATH="/usr/lib/cri-o-runc/sbin/runc"
if sudo dpkg -L cri-o-runc | grep -m 1 -q "$CRIO_RUNC_PATH"
then
echo "Linking $CRIO_RUNC_PATH to /usr/bin/runc for ease of testing."
sudo ln -f "$CRIO_RUNC_PATH" "/usr/bin/runc"
fi
# Stop disruption upon boot ASAP after booting
echo "Disabling all packaging activity on boot"
for filename in $(sudo ls -1 /etc/apt/apt.conf.d); do \
echo "Checking/Patching $filename"
sudo sed -i -r -e "s/$PERIODIC_APT_RE/"'\10"\;/' "/etc/apt/apt.conf.d/$filename"; done
bash $PACKER_BASE/ubuntu_packaging.sh
# Load installed environment right now (happens automatically in a new process)
source /usr/share/automation/environment
echo "Making Ubuntu kernel to enable cgroup swap accounting as it is not the default."
SEDCMD='s/^GRUB_CMDLINE_LINUX="(.*)"/GRUB_CMDLINE_LINUX="\1 cgroup_enable=memory swapaccount=1"/g'

10
contrib/cirrus/setup_environment.sh
View File

@ -43,16 +43,8 @@ case "${OS_RELEASE_ID}" in
fedora)
# All SELinux distros need this for systemd-in-a-container
setsebool container_manage_cgroup true
if [[ "$ADD_SECOND_PARTITION" == "true" ]]; then
bash "$SCRIPT_BASE/add_second_partition.sh"
fi
if [[ $OS_RELEASE_VER -le 31 ]]; then
warn "Switching io scheduler to 'deadline' to avoid RHBZ 1767539"
warn "aka https://bugzilla.kernel.org/show_bug.cgi?id=205447"
echo "mq-deadline" > /sys/block/sda/queue/scheduler
cat /sys/block/sda/queue/scheduler
fi
workaround_bfq_bug
if [[ "$ADD_SECOND_PARTITION" == "true" ]]; then
bash "$SCRIPT_BASE/add_second_partition.sh"