rework CI tests to test on VMs

This PR makes several key changes to our CI testing.  Firstly, we now test
podman on fedora 28, fedora 29, and centos VMS (rather than containers). Any
of these that having failing tests are not marked as required yet. We
still preserve the podman in podman and podman in docker tests as well and
they are marked as required.

The lint and validate work is now done on a openshift container.  We also
removed the rpm verification on papr and perform this test under the "images"
test on the openshift ci.

This PR exposes integration test fails on some of our OSs.  My expectation is we
will fix those in additional PRs and as they are fixed, we should be flipping
the boolean bit to required.

Signed-off-by: baude <bbaude@redhat.com>

Closes: #1492
Approved by: mheon

.copr/Makefile

@@ -2,7 +2,18 @@
 mkfile_path := $(abspath $(lastword $(MAKEFILE_LIST)))
 current_dir := $(notdir $(patsubst %/,%,$(dir $(mkfile_path))))
 outdir := $(CURDIR)
+topdir := $(CURDIR)/rpmbuild
+SHORT_COMMIT ?= $(shell git rev-parse --short=8 HEAD)
 
 srpm:
+	mkdir -p $(topdir)
 	sh $(current_dir)/prepare.sh
-	rpmbuild -bs -D "dist %{nil}" -D "_sourcedir build/" -D "_srcrpmdir $(outdir)" --nodeps contrib/spec/podman.spec
+	rpmbuild -bs -D "dist %{nil}" -D "_sourcedir build/" -D "_srcrpmdir $(outdir)" -D "_topdir $(topdir)" --nodeps contrib/spec/podman.spec
+
+build_binary:
+	mkdir -p $(topdir)
+	rpmbuild --rebuild -D "_rpmdir $(outdir)" -D "_topdir $(topdir)" $(outdir)/podman-*.git$(SHORT_COMMIT).src.rpm
+
+clean:
+	rm -fr rpms
+	rm -fr cri-o

.papr.sh

@@ -2,20 +2,89 @@
 set -xeuo pipefail
 
 export GOPATH=/go
-export PATH=$HOME/gopath/bin:$PATH
+export PATH=$HOME/gopath/bin:$PATH:$GOPATH/bin
-export GOSRC=/$GOPATH/src/github.com/containers/libpod
+export GOSRC=$GOPATH/src/github.com/containers/libpod
+
+DIST=${DIST:=""}
+pwd
+
+# -i install
+# -b build
+# -t integration test
+# -u unit test
+# -v validate
+# -p run python tests
+
+build=0
+install=0
+integrationtest=0
+unittest=0
+validate=0
+runpython=0
+options=0
+install_tools_made=0
+
+while getopts "biptuv" opt; do
+    case "$opt" in
+    b) build=1
+       options=1
+       ;;
+    i) install=1
+       options=1
+       ;;
+    p) runpython=1
+       options=1
+        ;;
+    t) integrationtest=1
+       options=1
+       ;;
+    u) unittest=1
+       options=1
+       ;;
+    v) validate=1
+       options=1
+       ;;
+    esac
+done
+
+# If no options are passed, do everything
+if [ $options -eq 0 ]; then
+    build=1
+    install=1
+    integrationtest=1
+    unittest=1
+    validate=1
+fi
+
+# Make Install tools function used by multiple sections below
+make_install_tools () {
+    # Only make the install tools once
+    if [ $install_tools_made -eq 0 ]; then
+        make install.tools TAGS="${TAGS}"
+    fi
+    install_tools_made=1
+}
+
+CONTAINER_RUNTIME=${CONTAINER_RUNTIME:-none}
+
+if [ "${CONTAINER_RUNTIME}" == "none" ]; then
+    mkdir -p /$GOPATH/src/github.com/containers/
+    mv /var/tmp/checkout $GOSRC
+    cd $GOSRC
+    pwd
+fi
 
 
+export TAGS="seccomp $($GOSRC/hack/btrfs_tag.sh) $($GOSRC/hack/libdm_tag.sh) $($GOSRC/hack/btrfs_installed_tag.sh) $($GOSRC/hack/ostree_tag.sh) $($GOSRC/hack/selinux_tag.sh)"
+
+# Validate
+if [ $validate -eq 1 ]; then
+    make_install_tools
     # PAPR adds a merge commit, for testing, which fails the
     # short-commit-subject validation test, so tell git-validate.sh to only check
     # up to, but not including, the merge commit.
     export GITVALIDATE_TIP=$(cd $GOSRC; git log -2 --pretty='%H' | tail -n 1)
-export TAGS="seccomp $($GOSRC/hack/btrfs_tag.sh) $($GOSRC/hack/libdm_tag.sh) $($GOSRC/hack/btrfs_installed_tag.sh) $($GOSRC/hack/ostree_tag.sh) $($GOSRC/hack/selinux_tag.sh)"
-
     make gofmt TAGS="${TAGS}"
-make localunit TAGS="${TAGS}"
-
-make install.tools TAGS="${TAGS}"
 
     # Only check lint and gitvalidation on more recent
     # distros with updated git and tooling
@@ -23,12 +92,41 @@ if [[ ${DIST} == "Fedora" ]]; then
         HEAD=$GITVALIDATE_TIP make -C $GOSRC .gitvalidation TAGS="${TAGS}"
         make lint
     fi
+fi
 
-# Make and install podman
+# Unit tests
-make TAGS="${TAGS}"
+if [ $unittest -eq 1 ]; then
-make TAGS="${TAGS}" install PREFIX=/usr ETCDIR=/etc
+    make localunit TAGS="${TAGS}"
+fi
+
+# Make Podman
+if [ $build -eq 1 ]; then
+    make_install_tools
+    make TAGS="${TAGS}" GOPATH=$GOPATH
+fi
+
+# Install Podman
+if [ $install -eq 1 ]; then
+    make_install_tools
+    make TAGS="${TAGS}" install.bin PREFIX=/usr ETCDIR=/etc
+    make TAGS="${TAGS}" install.man PREFIX=/usr ETCDIR=/etc
+    make TAGS="${TAGS}" install.cni PREFIX=/usr ETCDIR=/etc
+    make TAGS="${TAGS}" install.systemd PREFIX=/usr ETCDIR=/etc
+    if [ $runpython -eq 1 ]; then
+        make TAGS="${TAGS}" install.python PREFIX=/usr ETCDIR=/etc
+    fi
+
+fi
+
+# Run integration tests
+if [ $integrationtest -eq 1 ]; then
     make TAGS="${TAGS}" test-binaries
+    SKIP_USERNS=1 make varlink_generate GOPATH=/go
+    if [ $runpython -eq 1 ]; then
+        SKIP_USERNS=1 make clientintegration GOPATH=/go
+    fi
+    SKIP_USERNS=1 make ginkgo GOPATH=/go
+fi
+
 
-# Run the ginkgo integration tests
-SKIP_USERNS=1 GOPATH=/go make localintegration
 exit 0

.papr.yml

@@ -17,7 +17,8 @@ tests:
 artifacts:
   - build.log
 
-context: "FAH28"
+context: "FAH28 - Containerized (Podman in Podman)"
+
 ---
 
  host:
@@ -38,48 +39,134 @@ required: true
  timeout: 90m
 
  tests:
-  - sh .papr_prepare.sh
+   - CONTAINER_RUNTIME="docker" sh .papr_prepare.sh
 
  artifacts:
    - build.log
 
-context: "CAH smoketested"
+ context: "CAH 7-smoketested - Containerized (Podman in Docker)"
-
+
+---
+
+host:
+  distro: centos/7/cloud
+  specs:
+     ram: 8192
+     cpus: 4
+extra-repos:
+    - name: epel
+      metalink: https://mirrors.fedoraproject.org/metalink?repo=epel-7&arch=$basearch
+      gpgcheck: 0
+    - name: cri-o
+      baseurl: https://cbs.centos.org/repos/virt7-container-common-candidate/$basearch/os
+      gpgcheck: 0
+
+packages:
+    - btrfs-progs-devel
+    - glib2-devel
+    - glibc-devel
+    - glibc-static
+    - git
+    - go-md2man
+    - gpgme-devel
+    - libassuan-devel
+    - libgpg-error-devel
+    - libseccomp-devel
+    - libselinux-devel
+    - ostree-devel
+    - pkgconfig
+    - make
+    - nc
+    - go-compilers-golang-compiler
+    - podman
+
+required: true
+
+timeout: 90m
+
+tests:
+    - sed 's/^expand-check.*/expand-check=0/g' -i /etc/selinux/semanage.conf
+    - sh .papr.sh -b -i -t
+
+artifacts:
+  - build.log
+
+context: "CentOS 7 Cloud"
 
 ---
 
-inherit: true
 host:
     distro: fedora/28/cloud
     specs:
         ram: 8192
         cpus: 4
 packages:
+    - btrfs-progs-devel
+    - glib2-devel
+    - glibc-devel
+    - glibc-static
+    - git
+    - go-md2man
+    - gpgme-devel
+    - libassuan-devel
+    - libgpg-error-devel
+    - libseccomp-devel
+    - libselinux-devel
+    - ostree-devel
+    - pkgconfig
+    - make
+    - nc
+    - go-compilers-golang-compiler
     - podman
-    - buildah
+    - python3-varlink
+    - python3-dateutil
+    - python3-psutil
+
 tests:
     - sed 's/^expand-check.*/expand-check=0/g' -i /etc/selinux/semanage.conf
     - yum -y reinstall container-selinux
-    - CONTAINER_RUNTIME="podman" sh .papr_prepare.sh
+    - sh .papr.sh -b -i -t -p
+
 required: false
 
 timeout: 90m
-context: "Fedora fedora/28/cloud Podman"
+context: "Fedora 28 Cloud"
 
 ---
 
-container:
+host:
-    image: registry.fedoraproject.org/fedora:28
+    distro: fedora/29/cloud/pungi
-tests:
+    specs:
-    - sh contrib/build_rpm.sh
+        ram: 8192
-required: true
+        cpus: 4
-context: "Fedora RPM regressions"
+packages:
+    - btrfs-progs-devel
+    - glib2-devel
+    - glibc-devel
+    - glibc-static
+    - git
+    - go-md2man
+    - gpgme-devel
+    - libassuan-devel
+    - libgpg-error-devel
+    - libseccomp-devel
+    - libselinux-devel
+    - ostree-devel
+    - pkgconfig
+    - make
+    - nc
+    - go-compilers-golang-compiler
+    - podman
+    - python3-varlink
+    - python3-dateutil
+    - python3-psutil
 
----
-
-container:
-    image: registry.centos.org/centos:7
 tests:
-    - sh contrib/build_rpm.sh
+    - sed 's/^expand-check.*/expand-check=0/g' -i /etc/selinux/semanage.conf
-required: true
+    - yum -y reinstall container-selinux
-context: "CentOS RPM regressions"
+    - sh .papr.sh -b -i -t
+
+required: false
+
+timeout: 90m
+context: "Fedora 29 Cloud"

.papr_prepare.sh

@@ -14,4 +14,4 @@ fi
 ${CONTAINER_RUNTIME} build -t ${IMAGE} -f Dockerfile.${DIST} . 2>build.log
 
 # Run the tests
-${CONTAINER_RUNTIME} run --rm --privileged --net=host -v $PWD:/go/src/github.com/containers/libpod --workdir /go/src/github.com/containers/libpod -e CGROUP_MANAGER=cgroupfs -e PYTHON=$PYTHON -e STORAGE_OPTIONS="--storage-driver=vfs" -e CRIO_ROOT="/go/src/github.com/containers/libpod" -e PODMAN_BINARY="/usr/bin/podman" -e CONMON_BINARY="/usr/libexec/podman/conmon" -e DIST=$DIST $IMAGE sh .papr.sh
+${CONTAINER_RUNTIME} run --rm --privileged --net=host -v $PWD:/go/src/github.com/containers/libpod --workdir /go/src/github.com/containers/libpod -e CGROUP_MANAGER=cgroupfs -e PYTHON=$PYTHON -e STORAGE_OPTIONS="--storage-driver=vfs" -e CRIO_ROOT="/go/src/github.com/containers/libpod" -e PODMAN_BINARY="/usr/bin/podman" -e CONMON_BINARY="/usr/libexec/podman/conmon" -e DIST=$DIST -e CONTAINER_RUNTIME=$CONTAINER_RUNTIME $IMAGE sh .papr.sh

Makefile

@@ -18,7 +18,7 @@ ETCDIR ?= ${DESTDIR}/etc
 ETCDIR_LIBPOD ?= ${ETCDIR}/crio
 TMPFILESDIR ?= ${PREFIX}/lib/tmpfiles.d
 SYSTEMDDIR ?= ${PREFIX}/lib/systemd/system
-BUILDTAGS ?= seccomp $(shell hack/btrfs_tag.sh) $(shell hack/libdm_tag.sh) $(shell hack/btrfs_installed_tag.sh) $(shell hack/ostree_tag.sh) $(shell hack/selinux_tag.sh) $(shell hack/apparmor_tag.sh) varlink
+BUILDTAGS ?= seccomp $(shell hack/btrfs_tag.sh) $(shell hack/btrfs_installed_tag.sh) $(shell hack/ostree_tag.sh) $(shell hack/selinux_tag.sh) $(shell hack/apparmor_tag.sh) varlink exclude_graphdriver_devicemapper
 BUILDTAGS_CROSS ?= containers_image_openpgp containers_image_ostree_stub exclude_graphdriver_btrfs exclude_graphdriver_devicemapper exclude_graphdriver_overlay
 ifneq (,$(findstring varlink,$(BUILDTAGS)))
 	PODMAN_VARLINK_DEPENDENCIES = cmd/podman/varlink/iopodman.go
@@ -173,10 +173,9 @@ localunit: varlink_generate
 	$(GO) test -tags "$(BUILDTAGS)" -cover $(PACKAGES)
 
 ginkgo:
-	ginkgo -v test/e2e/
+	ginkgo -v -tags "$(BUILDTAGS)" -cover -flakeAttempts 3 -progress -trace -noColor test/e2e/.
 
-localintegration: varlink_generate test-binaries clientintegration
+localintegration: varlink_generate test-binaries clientintegration ginkgo
-	ginkgo -v -cover -flakeAttempts 3 -progress -trace -noColor test/e2e/.
 
 clientintegration:
 	$(MAKE) -C contrib/python/podman integration
@@ -268,7 +267,15 @@ uninstall:
 
 .PHONY: install.tools
 
-install.tools: .install.gitvalidation .install.gometalinter .install.md2man .install.easyjson
+install.tools: .install.gitvalidation .install.gometalinter .install.md2man .install.easyjson .install.ginkgo .install.gomega
+
+.install.gomega: .gopathok
+	$(GO) get github.com/onsi/gomega/...
+
+.install.ginkgo: .gopathok
+	if [ ! -x "$(GOBIN)/ginkgo" ]; then \
+		$(GO) get -u github.com/onsi/ginkgo/ginkgo; \
+	fi
 
 .install.gitvalidation: .gopathok
 	if [ ! -x "$(GOBIN)/git-validation" ]; then \
@@ -309,11 +316,11 @@ easyjson_generate: .gopathok libpod/container_easyjson.go libpod/pod_easyjson.go
 
 libpod/container_easyjson.go: libpod/container.go
 	rm -f libpod/container_easyjson.go
-	cd "$(GOPKGDIR)" && easyjson ./libpod/container.go
+	cd "$(GOPKGDIR)" && easyjson -build_tags "$(BUILDTAGS)" ./libpod/container.go
 
 libpod/pod_easyjson.go: libpod/pod.go
 	rm -f libpod/pod_easyjson.go
-	cd "$(GOPKGDIR)" && easyjson ./libpod/pod.go
+	cd "$(GOPKGDIR)" && easyjson -build_tags "$(BUILDTAGS)" ./libpod/pod.go
 
 .PHONY: install.libseccomp.sudo
 install.libseccomp.sudo:

contrib/spec/podman.spec.in

@@ -50,9 +50,7 @@ Source1: crio.tar.gz
 #ExclusiveArch:  %%{?go_arches:%%{go_arches}}%%{!?go_arches:%%{ix86} x86_64 aarch64 %%{arm}}
 ExclusiveArch: aarch64 %{arm} ppc64le s390x x86_64
 # If go_compiler is not set to 1, there is no virtual provide. Use golang instead.
-BuildRequires: %{?go_compiler:compiler(go-compiler)}%{!?go_compiler:golang}
 BuildRequires: btrfs-progs-devel
-BuildRequires: device-mapper-devel
 BuildRequires: glib2-devel
 BuildRequires: glibc-devel
 BuildRequires: glibc-static

test/e2e/libpod_suite_test.go

@@ -132,7 +132,7 @@ func PodmanCreate(tempDir string) PodmanTest {
 		podmanBinary = os.Getenv("PODMAN_BINARY")
 	}
 	conmonBinary := filepath.Join("/usr/libexec/podman/conmon")
-	altConmonBinary := "/usr/libexec/podman/conmon"
+	altConmonBinary := "/usr/libexec/crio/conmon"
 	if _, err := os.Stat(altConmonBinary); err == nil {
 		conmonBinary = altConmonBinary
 	}

test/e2e/run_test.go

@@ -50,10 +50,10 @@ var _ = Describe("Podman run", func() {
 
 	It("podman run a container based on local image with short options and args", func() {
 		// regression test for #714
-		session := podmanTest.Podman([]string{"run", ALPINE, "find", "/", "-name", "etc"})
+		session := podmanTest.Podman([]string{"run", ALPINE, "find", "/etc", "-name", "hosts"})
 		session.WaitWithDefaultTimeout()
 		Expect(session.ExitCode()).To(Equal(0))
-		match, _ := session.GrepString("/etc")
+		match, _ := session.GrepString("/etc/hosts")
 		Expect(match).Should(BeTrue())
 	})