opensource/podman
1
0
Fork 0
mirror of https://github.com/containers/podman.git synced 2025-05-22 09:36:57 +08:00
Code Issues Packages Projects Releases Wiki Activity

Cirrus: Reimplement release archive + upload

Browse Source 
The initial implementation was far more complicated than necessary.
Strip out the complexities in favor of a simpler and more direct
approach.

Signed-off-by: Chris Evich <cevich@redhat.com>
This commit is contained in:
Chris Evich
2019-08-01 07:31:04 -04:00
parent 8e46106f42
commit 370b1a887c
18 changed files with 326 additions and 292 deletions
Download Patch File Download Diff File Expand all files Collapse all files

70
.cirrus.yml
View File

@ -62,6 +62,10 @@ env:
GCE_SSH_USERNAME: cirrus-ci GCE_SSH_USERNAME: cirrus-ci
# Name where this repositories cloud resources are located # Name where this repositories cloud resources are located
GCP_PROJECT_ID: ENCRYPTED[7c80e728e046b1c76147afd156a32c1c57d4a1ac1eab93b7e68e718c61ca8564fc61fef815952b8ae0a64e7034b8fe4f] GCP_PROJECT_ID: ENCRYPTED[7c80e728e046b1c76147afd156a32c1c57d4a1ac1eab93b7e68e718c61ca8564fc61fef815952b8ae0a64e7034b8fe4f]
RELEASE_GCPJSON: ENCRYPTED[789d8f7e9a5972ce350fd8e60f1032ccbf4a35c3938b604774b711aad280e12c21faf10e25af1e0ba33597ffb9e39e46]
RELEASE_GCPNAME: ENCRYPTED[417d50488a4bd197bcc925ba6574de5823b97e68db1a17e3a5fde4bcf26576987345e75f8d9ea1c15a156b4612c072a1]
RELEASE_GCPROJECT: ENCRYPTED[7c80e728e046b1c76147afd156a32c1c57d4a1ac1eab93b7e68e718c61ca8564fc61fef815952b8ae0a64e7034b8fe4f]
# Default VM to use unless set or modified by task # Default VM to use unless set or modified by task
@ -336,9 +340,8 @@ testing_task:
unit_test_script: '$SCRIPT_BASE/unit_test.sh |& ${TIMESTAMP}' unit_test_script: '$SCRIPT_BASE/unit_test.sh |& ${TIMESTAMP}'
integration_test_script: '$SCRIPT_BASE/integration_test.sh |& ${TIMESTAMP}' integration_test_script: '$SCRIPT_BASE/integration_test.sh |& ${TIMESTAMP}'
system_test_script: '$SCRIPT_BASE/system_test.sh |& ${TIMESTAMP}' system_test_script: '$SCRIPT_BASE/system_test.sh |& ${TIMESTAMP}'
cache_release_archive_script: >- build_release_script: '$SCRIPT_BASE/build_release.sh |& ${TIMESTAMP}'
[[ "$TEST_REMOTE_CLIENT" == "false" ]] || \ upload_release_archive_script: '$SCRIPT_BASE/upload_release_archive.sh |& ${TIMESTAMP}'
$SCRIPT_BASE/cache_release_archive.sh |& ${TIMESTAMP}
on_failure: on_failure:
failed_branch_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_branch_failure.sh' failed_branch_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_branch_failure.sh'
@ -376,9 +379,6 @@ testing_crun_task:
unit_test_script: '$SCRIPT_BASE/unit_test.sh |& ${TIMESTAMP}' unit_test_script: '$SCRIPT_BASE/unit_test.sh |& ${TIMESTAMP}'
integration_test_script: '$SCRIPT_BASE/integration_test.sh |& ${TIMESTAMP}' integration_test_script: '$SCRIPT_BASE/integration_test.sh |& ${TIMESTAMP}'
system_test_script: '$SCRIPT_BASE/system_test.sh |& ${TIMESTAMP}' system_test_script: '$SCRIPT_BASE/system_test.sh |& ${TIMESTAMP}'
cache_release_archive_script: >-
[[ "$TEST_REMOTE_CLIENT" == "false" ]] || \
$SCRIPT_BASE/cache_release_archive.sh |& ${TIMESTAMP}
on_failure: on_failure:
failed_branch_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_branch_failure.sh' failed_branch_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_branch_failure.sh'
@ -459,14 +459,15 @@ special_testing_cross_task:
env: env:
matrix: matrix:
SPECIALMODE: 'windows' # See docs CROSS_PLATFORM: 'windows'
SPECIALMODE: 'darwin' CROSS_PLATFORM: 'darwin'
timeout_in: 20m timeout_in: 20m
networking_script: '${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/networking.sh' networking_script: '${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/networking.sh'
setup_environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}' setup_environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}'
cache_release_archive_script: '$SCRIPT_BASE/cache_release_archive.sh |& ${TIMESTAMP}' build_release_script: '$SCRIPT_BASE/build_release.sh |& ${TIMESTAMP}'
upload_release_archive_script: '$SCRIPT_BASE/upload_release_archive.sh |& ${TIMESTAMP}'
on_failure: on_failure:
failed_branch_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_branch_failure.sh' failed_branch_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_branch_failure.sh'
@ -615,6 +616,9 @@ verify_test_built_images_task:
integration_test_script: >- integration_test_script: >-
[[ "$PACKER_BUILDER_NAME" == "xfedora-30" ]] || \ [[ "$PACKER_BUILDER_NAME" == "xfedora-30" ]] || \
$SCRIPT_BASE/integration_test.sh |& ${TIMESTAMP} $SCRIPT_BASE/integration_test.sh |& ${TIMESTAMP}
build_release_script: >-
[[ "$PACKER_BUILDER_NAME" == "xfedora-30" ]] || \
'$SCRIPT_BASE/build_release.sh |& ${TIMESTAMP}'
system_test_script: >- system_test_script: >-
[[ "$PACKER_BUILDER_NAME" == "xfedora-30" ]] || \ [[ "$PACKER_BUILDER_NAME" == "xfedora-30" ]] || \
$SCRIPT_BASE/system_test.sh |& ${TIMESTAMP} $SCRIPT_BASE/system_test.sh |& ${TIMESTAMP}
@ -629,7 +633,7 @@ success_task:
# it blocks PRs from merging if a depends_on task fails # it blocks PRs from merging if a depends_on task fails
only_if: $CIRRUS_BRANCH != $DEST_BRANCH only_if: $CIRRUS_BRANCH != $DEST_BRANCH
# ignores any dependent task conditions, include everything except 'release' # ignores any dependent task conditions
depends_on: depends_on:
- "gating" - "gating"
- "vendor" - "vendor"
@ -660,49 +664,3 @@ success_task:
memory: 1 memory: 1
success_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/success.sh |& ${TIMESTAMP}' success_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/success.sh |& ${TIMESTAMP}'
release_task:
# Never do this when building images
only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\*\*\*\s*CIRRUS:\s*TEST\s*IMAGES\s*\*\*\*.*'
# TODO: Uncomment both to not affect pass/fail status of entire job?
# allow_failures: $CI == "true"
# skip_notifications: $CI == "true"
# Must include everything (YAML anchor/alias cannot be used here)
depends_on:
- "gating"
- "vendor"
- "varlink_api"
- "build_each_commit"
- "build_without_cgo"
- "meta"
- "image_prune"
- "testing"
- "testing_crun"
- "special_testing_rootless"
- "special_testing_in_podman"
- "special_testing_cgroupv2"
- "special_testing_cross"
- "special_testing_endpoint"
- "test_build_cache_images"
- "test_building_snap"
- "verify_test_built_images"
- "success"
gce_instance:
image_name: "${IMAGE_BUILDER_CACHE_IMAGE_NAME}"
timeout_in: 30m
env:
GCPJSON: ENCRYPTED[789d8f7e9a5972ce350fd8e60f1032ccbf4a35c3938b604774b711aad280e12c21faf10e25af1e0ba33597ffb9e39e46]
GCPNAME: ENCRYPTED[417d50488a4bd197bcc925ba6574de5823b97e68db1a17e3a5fde4bcf26576987345e75f8d9ea1c15a156b4612c072a1]
GCPROJECT: ENCRYPTED[7c80e728e046b1c76147afd156a32c1c57d4a1ac1eab93b7e68e718c61ca8564fc61fef815952b8ae0a64e7034b8fe4f]
uncache_release_archives_script: '$SCRIPT_BASE/uncache_release_archives.sh |& ${TIMESTAMP}'
on_failure:
failed_branch_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_branch_failure.sh'

3
.gitignore vendored
View File

@ -20,6 +20,7 @@ __pycache__
/cmd/podman/varlink/iopodman.go /cmd/podman/varlink/iopodman.go
.gopathok .gopathok
test/e2e/e2e.coverprofile test/e2e/e2e.coverprofile
/podman*zip release.txt
podman-remote*.zip
podman*.tar.gz podman*.tar.gz
.idea* .idea*

106
Makefile
View File

@ -81,13 +81,12 @@ LIBSECCOMP_COMMIT := release-2.3
# caller may override in special circumstances if needed. # caller may override in special circumstances if needed.
GINKGOTIMEOUT ?= -timeout=90m GINKGOTIMEOUT ?= -timeout=90m
RELEASE_VERSION ?= $(shell git fetch --tags && git describe HEAD 2> /dev/null) RELEASE_VERSION ?= $(shell hack/get_release_info.sh VERSION)
RELEASE_NUMBER ?= $(shell echo $(RELEASE_VERSION) | sed 's/-.*//') RELEASE_NUMBER ?= $(shell hack/get_release_info.sh NUMBER)
RELEASE_DIST ?= $(shell ( source /etc/os-release; echo $$ID )) RELEASE_DIST ?= $(shell hack/get_release_info.sh DIST)
RELEASE_DIST_VER ?= $(shell ( source /etc/os-release; echo $$VERSION_ID | cut -d '.' -f 1)) RELEASE_DIST_VER ?= $(shell hack/get_release_info.sh DIST_VER)
RELEASE_ARCH ?= $(shell go env GOARCH 2> /dev/null) RELEASE_ARCH ?= $(shell hack/get_release_info.sh ARCH)
RELEASE_BASENAME := $(shell basename $(PROJECT)) RELEASE_BASENAME := $(shell hack/get_release_info.sh BASENAME)
# If GOPATH not specified, use one in the local directory # If GOPATH not specified, use one in the local directory
ifeq ($(GOPATH),) ifeq ($(GOPATH),)
@ -164,11 +163,9 @@ podman: .gopathok $(PODMAN_VARLINK_DEPENDENCIES) ## Build with podman
podman-remote: .gopathok $(PODMAN_VARLINK_DEPENDENCIES) ## Build with podman on remote environment podman-remote: .gopathok $(PODMAN_VARLINK_DEPENDENCIES) ## Build with podman on remote environment
$(GO_BUILD) $(BUILDFLAGS) -gcflags '$(GCFLAGS)' -asmflags '$(ASMFLAGS)' -ldflags '$(LDFLAGS_PODMAN)' -tags "$(BUILDTAGS) remoteclient" -o bin/$@ $(PROJECT)/cmd/podman $(GO_BUILD) $(BUILDFLAGS) -gcflags '$(GCFLAGS)' -asmflags '$(ASMFLAGS)' -ldflags '$(LDFLAGS_PODMAN)' -tags "$(BUILDTAGS) remoteclient" -o bin/$@ $(PROJECT)/cmd/podman
podman-remote-darwin: .gopathok $(PODMAN_VARLINK_DEPENDENCIES) ## Build with podman on remote OSX environment podman-remote-%: .gopathok $(PODMAN_VARLINK_DEPENDENCIES) ## Build podman for a specific GOOS
CGO_ENABLED=0 GOOS=darwin $(GO_BUILD) -gcflags '$(GCFLAGS)' -asmflags '$(ASMFLAGS)' -ldflags '$(LDFLAGS_PODMAN)' -tags "remoteclient containers_image_openpgp exclude_graphdriver_devicemapper" -o bin/$@ $(PROJECT)/cmd/podman $(eval BINSFX := $(shell test "$*" != "windows" || echo ".exe"))
CGO_ENABLED=0 GOOS=$* $(GO_BUILD) -gcflags '$(GCFLAGS)' -asmflags '$(ASMFLAGS)' -ldflags '$(LDFLAGS_PODMAN)' -tags "remoteclient containers_image_openpgp exclude_graphdriver_devicemapper" -o bin/$@$(BINSFX) $(PROJECT)/cmd/podman
podman-remote-windows: .gopathok $(PODMAN_VARLINK_DEPENDENCIES) ## Build with podman for a remote windows environment
CGO_ENABLED=0 GOOS=windows $(GO_BUILD) -gcflags '$(GCFLAGS)' -asmflags '$(ASMFLAGS)' -ldflags '$(LDFLAGS_PODMAN)' -tags "remoteclient containers_image_openpgp exclude_graphdriver_devicemapper" -o bin/$@.exe $(PROJECT)/cmd/podman
local-cross: $(CROSS_BUILD_TARGETS) ## Cross local compilation local-cross: $(CROSS_BUILD_TARGETS) ## Cross local compilation
@ -182,8 +179,9 @@ clean: ## Clean artifacts
rm -rf \ rm -rf \
.gopathok \ .gopathok \
_output \ _output \
podman*.zip \ release.txt
podman*.tar.gz \ $(wildcard podman-remote*.zip) \
$(wildcard podman*.tar.gz) \
bin \ bin \
build \ build \
docs/remote \ docs/remote \
@ -300,23 +298,6 @@ vagrant-check:
binaries: varlink_generate podman podman-remote ## Build podman binaries: varlink_generate podman podman-remote ## Build podman
# Zip archives are supported on all platforms + allows embedding metadata
podman.zip: binaries docs
$(eval TMPDIR := $(shell mktemp -d -p '' $@_XXXX))
test -n "$(TMPDIR)"
$(MAKE) install "DESTDIR=$(TMPDIR)" "PREFIX=$(TMPDIR)/usr"
# Encoded RELEASE_INFO format depended upon by CI tooling
# X-RELEASE-INFO format depended upon by CI tooling
cd "$(TMPDIR)" && echo \
"X-RELEASE-INFO: $(RELEASE_BASENAME) $(RELEASE_VERSION) $(RELEASE_DIST) $(RELEASE_DIST_VER) $(RELEASE_ARCH)" | \
zip --recurse-paths --archive-comment "$(CURDIR)/$@" "./"
-rm -rf "$(TMPDIR)"
podman-remote-%.zip: podman-remote-%
# Don't label darwin/windows cros-compiles with local distribution & version
echo "X-RELEASE-INFO: podman-remote $(RELEASE_VERSION) $* cc $(RELEASE_ARCH)" | \
zip --archive-comment "$(CURDIR)/$@" ./bin/$<*
install.catatonit: install.catatonit:
./hack/install_catatonit.sh ./hack/install_catatonit.sh
@ -333,19 +314,58 @@ docs: $(MANPAGES) ## Generate documentation
install-podman-remote-docs: docs install-podman-remote-docs: docs
@(cd docs; ./podman-remote.sh ./remote) @(cd docs; ./podman-remote.sh ./remote)
# When publishing releases include critical build-time details
.PHONY: release.txt
release.txt:
# X-RELEASE-INFO format depended upon by automated tooling
echo -n "X-RELEASE-INFO:" > "$@"
for field in "$(RELEASE_BASENAME)" "$(RELEASE_VERSION)" \
"$(RELEASE_DIST)" "$(RELEASE_DIST_VER)" "$(RELEASE_ARCH)"; do \
echo -n " $$field"; done >> "$@"
echo "" >> "$@"
brew-pkg: install-podman-remote-docs podman-remote-darwin podman-$(RELEASE_NUMBER).tar.gz: binaries docs release.txt
@mkdir -p ./brew $(eval TMPDIR := $(shell mktemp -d -p '' podman_XXXX))
@cp ./bin/podman-remote-darwin ./brew/podman $(eval SUBDIR := podman-$(RELEASE_NUMBER))
@cp -r ./docs/remote ./brew/docs/ mkdir -p "$(TMPDIR)/$(SUBDIR)"
@cp docs/podman-remote.1 ./brew/docs/podman.1 $(MAKE) install.bin install.man install.cni install.systemd "DESTDIR=$(TMPDIR)/$(SUBDIR)" "PREFIX=/usr"
@cp docs/podman-remote.conf.5 ./brew/docs/podman-remote.conf.5 # release.txt location and content depended upon by automated tooling
@sed -i 's/podman\\*-remote/podman/g' ./brew/docs/podman.1 cp release.txt "$(TMPDIR)/"
@sed -i 's/Podman\\*-remote/Podman\ for\ Mac/g' ./brew/docs/podman.1 tar -czvf $@ --xattrs -C "$(TMPDIR)" "./release.txt" "./$(SUBDIR)"
@sed -i 's/podman\.conf/podman\-remote\.conf/g' ./brew/docs/podman.1 -rm -rf "$(TMPDIR)"
@sed -i 's/A\ remote\ CLI\ for\ Podman\:\ //g' ./brew/docs/podman.1
tar -czvf podman-${RELEASE_NUMBER}.tar.gz ./brew # Must call make in-line: Dependency-spec. w/ wild-card also consumes variable value.
@rm -rf ./brew podman-remote-$(RELEASE_NUMBER)-%.zip:
$(MAKE) podman-remote-$* install-podman-remote-docs release.txt \
RELEASE_BASENAME=$(shell hack/get_release_info.sh REMOTENAME) \
RELEASE_DIST=$* RELEASE_DIST_VER="-"
$(eval TMPDIR := $(shell mktemp -d -p '' $podman_remote_XXXX))
$(eval SUBDIR := podman-$(RELEASE_VERSION))
$(eval BINSFX := $(shell test "$*" != "windows" || echo ".exe"))
mkdir -p "$(TMPDIR)/$(SUBDIR)"
# release.txt location and content depended upon by automated tooling
cp release.txt "$(TMPDIR)/"
cp ./bin/podman-remote-$*$(BINSFX) "$(TMPDIR)/$(SUBDIR)/podman$(BINSFX)"
cp -r ./docs/remote "$(TMPDIR)/$(SUBDIR)/docs/"
$(eval DOCFILE := $(TMPDIR)/$(SUBDIR)/docs/podman.1)
cp docs/podman-remote.1 "$(DOCFILE)"
sed -i 's/podman\\*-remote/podman/g' "$(DOCFILE)"
sed -i 's/Podman\\*-remote/Podman\ for\ $*/g' "$(DOCFILE)"
sed -i 's/podman\.conf/podman\-remote\.conf/g' "$(DOCFILE)"
sed -i 's/A\ remote\ CLI\ for\ Podman\:\ //g' "$(DOCFILE)"
cd "$(TMPDIR)" && \
zip --recurse-paths "$(CURDIR)/$@" "./release.txt" "./"
-rm -rf "$(TMPDIR)"
.PHONY: podman-release
podman-release:
rm -f release.txt
$(MAKE) podman-$(RELEASE_NUMBER).tar.gz
.PHONY: podman-remote-%-release
podman-remote-%-release:
rm -f release.txt
$(MAKE) podman-remote-$(RELEASE_NUMBER)-$*.zip
docker-docs: docs docker-docs: docs
(cd docs; ./dckrman.sh *.1) (cd docs; ./dckrman.sh *.1)

11
README.md
View File

@ -8,14 +8,9 @@ popularized by Kubernetes. Libpod also contains the Pod Manager tool `(Podman)`
* [Latest Version: 1.4.4](https://github.com/containers/libpod/releases/latest) * [Latest Version: 1.4.4](https://github.com/containers/libpod/releases/latest)
* [Continuous Integration:](contrib/cirrus/README.md) [![Build Status](https://api.cirrus-ci.com/github/containers/libpod.svg)](https://cirrus-ci.com/github/containers/libpod/master) * [Continuous Integration:](contrib/cirrus/README.md) [![Build Status](https://api.cirrus-ci.com/github/containers/libpod.svg)](https://cirrus-ci.com/github/containers/libpod/master)
* [GoDoc: ![GoDoc](https://godoc.org/github.com/containers/libpod/libpod?status.svg)](https://godoc.org/github.com/containers/libpod/libpod) * [GoDoc: ![GoDoc](https://godoc.org/github.com/containers/libpod/libpod?status.svg)](https://godoc.org/github.com/containers/libpod/libpod)
* Latest, automatic binary release downloads: * Automated continuous release downloads (including remote-client):
* Master Branch: * Master Branch: [https://storage.cloud.google.com/libpod-master-releases/](https://storage.cloud.google.com/libpod-master-releases/)
* [Fedora 30](https://storage.cloud.google.com/libpod-master-releases/libpod-master-fedora-30-amd64.zip) * Pull-requests: [https://storage.cloud.google.com/libpod-pr-releases/](https://storage.cloud.google.com/libpod-pr-releases/)
* [Fedora 29](https://storage.cloud.google.com/libpod-master-releases/libpod-master-fedora-29-amd64.zip)
* [Ubuntu 18](https://storage.cloud.google.com/libpod-master-releases/libpod-master-ubuntu-18-amd64.zip)
* [Ubuntu 19](https://storage.cloud.google.com/libpod-master-releases/libpod-master-ubuntu-19-amd64.zip)
* [Windows Client](https://storage.cloud.google.com/libpod-master-releases/podman-remote-master-windows-cc-amd64.zip)
* [MacOS Client](https://storage.cloud.google.com/libpod-master-releases/podman-remote-master-darwin-cc-amd64.zip)
## Overview and scope ## Overview and scope

30
contrib/cirrus/build_release.sh Executable file
View File

@ -0,0 +1,30 @@
#!/bin/bash
source $(dirname $0)/lib.sh
req_env_var TEST_REMOTE_CLIENT OS_RELEASE_ID GOSRC
cd $GOSRC
if [[ "$TEST_REMOTE_CLIENT" == "true" ]] && [[ -z "$CROSS_PLATFORM" ]]
then
CROSS_PLATFORM=linux
fi
if [[ -n "$CROSS_PLATFORM" ]]
then
echo "Compiling podman-remote release archive for ${CROSS_PLATFORM}"
case "$CROSS_PLATFORM" in
linux) ;&
windows) ;&
darwin)
make podman-remote-${CROSS_PLATFORM}-release
;;
*)
die 1 "Unknown/unsupported cross-compile platform '$CROSS_PLATFORM'"
;;
esac
else
echo "Compiling release archive for $OS_RELEASE_ID"
make podman-release
fi

140
contrib/cirrus/cache_release_archive.sh
View File

@ -1,140 +0,0 @@
#!/bin/bash
set -eo pipefail
source $(dirname $0)/lib.sh
req_env_var GOSRC
RELEASE_ARCHIVE_NAMES=""
handle_archive() { # Assumed to be called with set +e
TASK_NUMBER=$1
PR_OR_BRANCH=$2
CACHE_URL=$3
ARCHIVE_NAME="$(basename $CACHE_URL)"
req_env_var TASK_NUMBER PR_OR_BRANCH CACHE_URL ARCHIVE_NAME
cd /tmp
curl -sO "$CACHE_URL" || return $(warn 0 "Couldn't download file, skipping.")
[[ -r "/tmp/$ARCHIVE_NAME" ]] || return $(warn 0 "Unreadable archive '/tmp/$ARCHIVE_NAME', skipping.")
ZIPCOMMENT=$(unzip -qqz "$ARCHIVE_NAME" 2>/dev/null) # noisy bugger
if [[ "$?" -ne "0" ]] || [[ -z "$ZIPCOMMENT" ]]
then
return $(warn 0 "Could not unzip metadata from downloaded '/tmp/$ARCHIVE_NAME', skipping.")
fi
RELEASE_INFO=$(echo "$ZIPCOMMENT" | grep -m 1 'X-RELEASE-INFO:' | sed -r -e 's/X-RELEASE-INFO:\s*(.+)/\1/')
if [[ "$?" -ne "0" ]] || [[ -z "$RELEASE_INFO" ]]
then
return $(warn 0 "Metadata empty or invalid: '$ZIPCOMMENT', skipping.")
fi
# e.g. libpod v1.3.1-166-g60df124e fedora 29 amd64
# or libpod v1.3.1-166-g60df124e amd64
FIELDS="RELEASE_BASENAME RELEASE_VERSION RELEASE_DIST RELEASE_DIST_VER RELEASE_ARCH"
read $FIELDS <<< $RELEASE_INFO
for f in $FIELDS
do
[[ -n "${!f}" ]] || return $(warn 0 "Expecting $f to be non-empty in metadata: '$RELEASE_INFO', skipping.")
done
echo -n "Preparing $RELEASE_BASENAME archive: "
# Drop version number to enable "latest" representation
# (version available w/in zip-file comment)
RELEASE_ARCHIVE_NAME="${RELEASE_BASENAME}-${PR_OR_BRANCH}-${RELEASE_DIST}-${RELEASE_DIST_VER}-${RELEASE_ARCH}.zip"
# Allow uploading all gathered files in parallel, later with gsutil.
mv -v "$ARCHIVE_NAME" "/$RELEASE_ARCHIVE_NAME"
RELEASE_ARCHIVE_NAMES="$RELEASE_ARCHIVE_NAMES $RELEASE_ARCHIVE_NAME"
}
make_release() {
ARCHIVE_NAME="$1"
req_env_var ARCHIVE_NAME
# There's no actual testing of windows/darwin targets yet
# but we still want to cross-compile and publish binaries
if [[ "$SPECIALMODE" == "windows" ]] || [[ "$SPECIALMODE" == "darwin" ]]
then
RELFILE="podman-remote-${SPECIALMODE}.zip"
elif [[ "$SPECIALMODE" == "none" ]]
then
RELFILE="podman.zip"
else
die 55 "$(basename $0) unable to handle \$SPECIALMODE=$SPECIALMODE for $ARCHIVE_NAME"
fi
echo "Calling make $RELFILE"
cd $GOSRC
make "$RELFILE"
echo "Renaming archive so it can be identified/downloaded for publishing"
mv -v "$RELFILE" "$ARCHIVE_NAME"
echo "Success!"
}
[[ "$CI" == "true" ]] || \
die 56 "$0 requires a Cirrus-CI cross-task cache to function"
cd $GOSRC
# Same script re-used for both uploading and downloading to avoid duplication
if [[ "$(basename $0)" == "cache_release_archive.sh" ]]
then
# ref: https://cirrus-ci.org/guide/writing-tasks/#environment-variables
req_env_var CI_NODE_INDEX CIRRUS_BUILD_ID
# Use unique names for uncache_release_archives.sh to find/download them all
ARCHIVE_NAME="build-${CIRRUS_BUILD_ID}-task-${CI_NODE_INDEX}.zip"
make_release "$ARCHIVE_NAME"
# ref: https://cirrus-ci.org/guide/writing-tasks/#http-cache
URL="http://$CIRRUS_HTTP_CACHE_HOST/${ARCHIVE_NAME}"
echo "Uploading $ARCHIVE_NAME to Cirrus-CI cache at $URL"
curl -s -X POST --data-binary "@$ARCHIVE_NAME" "$URL"
elif [[ "$(basename $0)" == "uncache_release_archives.sh" ]]
then
req_env_var CIRRUS_BUILD_ID CI_NODE_TOTAL GCPJSON GCPNAME GCPROJECT
[[ "${CI_NODE_INDEX}" -eq "$[CI_NODE_TOTAL-1]" ]] || \
die 0 "WARNING: This task depends on cache data from other tasks, otherwise it is a no-op."
if [[ -n "$CIRRUS_PR" ]]
then
PR_OR_BRANCH="pr$CIRRUS_PR"
BUCKET="libpod-pr-releases"
elif [[ -n "$CIRRUS_BRANCH" ]]
then
PR_OR_BRANCH="$CIRRUS_BRANCH"
BUCKET="libpod-$CIRRUS_BRANCH-releases"
else
die 10 "Expecting either \$CIRRUS_PR or \$CIRRUS_BRANCH to be non-empty."
fi
echo "Blindly downloading Cirrus-CI cache files for task (some will fail)."
set +e # Don't stop looping until all task's cache is attempted
for (( task_number = 0 ; task_number < $CI_NODE_TOTAL ; task_number++ ))
do
ARCHIVE_NAME="build-${CIRRUS_BUILD_ID}-task-${task_number}.zip"
URL="http://$CIRRUS_HTTP_CACHE_HOST/${ARCHIVE_NAME}"
echo "Attempting to download cached archive from $URL"
handle_archive "$task_number" "$PR_OR_BRANCH" "$URL"
echo "----------------------------------------"
done
set -e
[[ -n "$RELEASE_ARCHIVE_NAMES" ]] || \
die 67 "Error: No release archives found in CI cache, expecting at least one."
echo "Preparing to upload release archives."
gcloud config set project "$GCPROJECT"
echo "$GCPJSON" > /tmp/gcp.json
gcloud auth activate-service-account --key-file=/tmp/gcp.json
rm /tmp/gcp.json
# handle_archive() placed all uploadable files under /
gsutil -m cp /*.zip "gs://$BUCKET" # Upload in parallel
echo "Successfully uploaded archives:"
for ARCHIVE_NAME in $RELEASE_ARCHIVE_NAMES
do
echo " https://storage.cloud.google.com/$BUCKET/$ARCHIVE_NAME"
done
echo "These will remain available until automatic pruning by bucket policy."
else
die 9 "I don't know what to do when called $0"
fi

39
contrib/cirrus/cirrus_yaml_test.py
View File

@ -26,7 +26,6 @@ class TestCaseBase(unittest.TestCase):
class TestDependsOn(TestCaseBase): class TestDependsOn(TestCaseBase):
ALL_TASK_NAMES = None ALL_TASK_NAMES = None
SUCCESS_RELEASE = set(['success', 'release'])
def setUp(self): def setUp(self):
super().setUp() super().setUp()
@ -34,34 +33,22 @@ class TestDependsOn(TestCaseBase):
for key, _ in self.CIRRUS_YAML.items() for key, _ in self.CIRRUS_YAML.items()
if key.endswith('_task')]) if key.endswith('_task')])
def test_dicts(self): def test_00_dicts(self):
"""Expected dictionaries are present and non-empty""" """Expected dictionaries are present and non-empty"""
for name in ('success_task', 'release_task'): self.assertIn('success_task', self.CIRRUS_YAML)
# tests all names then show specific failures self.assertIn('success_task'.replace('_task', ''), self.ALL_TASK_NAMES)
with self.subTest(name=name): self.assertIn('depends_on', self.CIRRUS_YAML['success_task'])
self.assertIn(name, self.CIRRUS_YAML) self.assertGreater(len(self.CIRRUS_YAML['success_task']['depends_on']), 0)
self.assertIn(name.replace('_task', ''), self.ALL_TASK_NAMES)
self.assertIn('depends_on', self.CIRRUS_YAML[name])
self.assertGreater(len(self.CIRRUS_YAML[name]['depends_on']), 0)
def _check_dep(self, name, task_name, deps): def test_01_depends(self):
# name includes '_task' suffix, task_name does not """Success task depends on all other tasks"""
msg=('Please add "{0}" to the "depends_on" list in "{1}"' success_deps = set(self.CIRRUS_YAML['success_task']['depends_on'])
"".format(task_name, name)) for task_name in self.ALL_TASK_NAMES - set(['success']):
self.assertIn(task_name, deps, msg=msg) with self.subTest(task_name=task_name):
msg=('Please add "{0}" to the "depends_on" list in "success_task"'
"".format(task_name))
self.assertIn(task_name, success_deps, msg=msg)
def test_depends(self):
"""Success and Release tasks depend on all other tasks"""
for name in ('success_task', 'release_task'):
deps = set(self.CIRRUS_YAML[name]['depends_on'])
for task_name in self.ALL_TASK_NAMES - self.SUCCESS_RELEASE:
with self.subTest(name=name, task_name=task_name):
self._check_dep(name, task_name, deps)
def test_release(self):
"""Release task must always execute last"""
deps = set(self.CIRRUS_YAML['release_task']['depends_on'])
self._check_dep('release_task', 'success', deps)
if __name__ == "__main__": if __name__ == "__main__":

4
contrib/cirrus/integration_test.sh
View File

@ -65,10 +65,6 @@ case "$SPECIALMODE" in
make local${TESTSUITE} make local${TESTSUITE}
fi fi
;; ;;
windows) ;& # for podman-remote building only
darwin)
warn '' "No $SPECIALMODE remote client integration tests configured"
;;
*) *)
die 110 "Unsupported \$SPECIALMODE: $SPECIALMODE" die 110 "Unsupported \$SPECIALMODE: $SPECIALMODE"
esac esac

4
contrib/cirrus/lib.sh
View File

@ -64,6 +64,8 @@ export PRIOR_FEDORA_BASE_IMAGE="fedora-cloud-base-29-1-2-1559164849"
export BUILT_IMAGE_SUFFIX="${BUILT_IMAGE_SUFFIX:--$CIRRUS_REPO_NAME-${CIRRUS_BUILD_ID}}" export BUILT_IMAGE_SUFFIX="${BUILT_IMAGE_SUFFIX:--$CIRRUS_REPO_NAME-${CIRRUS_BUILD_ID}}"
# IN_PODMAN container image # IN_PODMAN container image
IN_PODMAN_IMAGE="quay.io/libpod/in_podman:latest" IN_PODMAN_IMAGE="quay.io/libpod/in_podman:latest"
# Image for uploading releases
UPLDREL_IMAGE="quay.io/libpod/upldrel:latest"
# Avoid getting stuck waiting for user input # Avoid getting stuck waiting for user input
export DEBIAN_FRONTEND="noninteractive" export DEBIAN_FRONTEND="noninteractive"
@ -76,7 +78,7 @@ BIGTO="timeout_attempt_delay_command 300s 5 30s"
# Safe env. vars. to transfer from root -> $ROOTLESS_USER (go env handled separetly) # Safe env. vars. to transfer from root -> $ROOTLESS_USER (go env handled separetly)
ROOTLESS_ENV_RE='(CIRRUS_.+)|(ROOTLESS_.+)|(.+_IMAGE.*)|(.+_BASE)|(.*DIRPATH)|(.*FILEPATH)|(SOURCE.*)|(DEPEND.*)|(.+_DEPS_.+)|(OS_REL.*)|(.+_ENV_RE)|(TRAVIS)|(CI.+)|(TEST_REMOTE.*)' ROOTLESS_ENV_RE='(CIRRUS_.+)|(ROOTLESS_.+)|(.+_IMAGE.*)|(.+_BASE)|(.*DIRPATH)|(.*FILEPATH)|(SOURCE.*)|(DEPEND.*)|(.+_DEPS_.+)|(OS_REL.*)|(.+_ENV_RE)|(TRAVIS)|(CI.+)|(TEST_REMOTE.*)'
# Unsafe env. vars for display # Unsafe env. vars for display
SECRET_ENV_RE='(IRCID)|(ACCOUNT)|(^GC[EP]..+)|(SSH)' SECRET_ENV_RE='(IRCID)|(ACCOUNT)|(GC[EP]..+)|(SSH)'
# Names of systemd units which should never be running # Names of systemd units which should never be running
EVIL_UNITS="cron crond atd apt-daily-upgrade apt-daily fstrim motd-news systemd-tmpfiles-clean" EVIL_UNITS="cron crond atd apt-daily-upgrade apt-daily fstrim motd-news systemd-tmpfiles-clean"

5
contrib/cirrus/setup_environment.sh
View File

@ -66,7 +66,8 @@ case "$SPECIALMODE" in
remove_packaged_podman_files # we're building from source remove_packaged_podman_files # we're building from source
;; ;;
none) none)
remove_packaged_podman_files [[ -n "$CROSS_PLATFORM" ]] || \
remove_packaged_podman_files
;; ;;
endpoint) endpoint)
remove_packaged_podman_files remove_packaged_podman_files
@ -88,8 +89,6 @@ case "$SPECIALMODE" in
in_podman) # Assumed to be Fedora in_podman) # Assumed to be Fedora
$SCRIPT_BASE/setup_container_environment.sh $SCRIPT_BASE/setup_container_environment.sh
;; ;;
windows) ;& # for podman-remote building only
darwin) ;;
*) *)
die 111 "Unsupported \$SPECIALMODE: $SPECIALMODE" die 111 "Unsupported \$SPECIALMODE: $SPECIALMODE"
esac esac

1
contrib/cirrus/uncache_release_archives.sh
View File

@ -1 +0,0 @@
cache_release_archive.sh

4
contrib/cirrus/unit_test.sh
View File

@ -16,10 +16,6 @@ case "$SPECIALMODE" in
none) none)
make make
;; ;;
windows) ;&
darwin)
make podman-remote-$SPECIALMODE
;;
*) *)
die 109 "Unsupported \$SPECIAL_MODE: $SPECIALMODE" die 109 "Unsupported \$SPECIAL_MODE: $SPECIALMODE"
esac esac

52
contrib/cirrus/upload_release_archive.sh Executable file
View File

@ -0,0 +1,52 @@
#!/bin/bash
set -eo pipefail
source $(dirname $0)/lib.sh
req_env_var CI UPLDREL_IMAGE CIRRUS_BUILD_ID GOSRC RELEASE_GCPJSON RELEASE_GCPNAME RELEASE_GCPROJECT
[[ "$CI" == "true" ]] || \
die 56 "$0 must be run under Cirrus-CI to function"
unset PR_OR_BRANCH BUCKET
if [[ -n "$CIRRUS_PR" ]]
then
PR_OR_BRANCH="pr$CIRRUS_PR"
BUCKET="libpod-pr-releases"
elif [[ -n "$CIRRUS_BRANCH" ]]
then
PR_OR_BRANCH="$CIRRUS_BRANCH"
BUCKET="libpod-$CIRRUS_BRANCH-releases"
else
die 1 "Expecting either \$CIRRUS_PR or \$CIRRUS_BRANCH to be non-empty."
fi
# Functional local podman required for uploading a release
cd $GOSRC
[[ -n "$(type -P podman)" ]] || \
make install || \
die 57 "$0 requires working podman binary on path to function"
TMPF=$(mktemp -p '' $(basename $0)_XXXX.json)
trap "rm -f $TMPF" EXIT
set +x
echo "$RELEASE_GCPJSON" > "$TMPF"
unset RELEASE_GCPJSON
cd $GOSRC
for filename in $(ls -1 *.tar.gz *.zip)
do
echo "Running podman ... $UPLDREL_IMAGE $filename"
podman run -i --rm \
-e "GCPNAME=$RELEASE_GCPNAME" \
-e "GCPPROJECT=$RELEASE_GCPROJECT" \
-e "GCPJSON_FILEPATH=$TMPF" \
-e "REL_ARC_FILEPATH=/tmp/$filename" \
-e "PR_OR_BRANCH=$PR_OR_BRANCH" \
-e "BUCKET=$BUCKET" \
--security-opt label=disable \
-v "$TMPF:$TMPF:ro" \
-v "$GOSRC/$filename:/tmp/$filename:ro" \
$UPLDREL_IMAGE
done

15
contrib/imgts/lib_entrypoint.sh
View File

@ -35,10 +35,15 @@ req_env_var() {
gcloud_init() { gcloud_init() {
set +xe set +xe
TMPF=$(mktemp -p '' .$(uuidgen)XXXX) if [[ -n "$1" ]] && [[ -r "$1" ]]
trap "rm -f $TMPF" EXIT then
echo "$GCPJSON" > $TMPF && \ TMPF="$1"
$GCLOUD auth activate-service-account --project "$GCPPROJECT" --key-file=$TMPF || \ else
TMPF=$(mktemp -p '' .$(uuidgen)_XXXX.json)
trap "rm -f $TMPF &> /dev/null" EXIT
echo "$GCPJSON" > $TMPF
fi
$GCLOUD auth activate-service-account --project="$GCPPROJECT" --key-file="$TMPF" || \
die 5 FATAL auth die 5 FATAL auth
rm -f $TMPF rm -f $TMPF &> /dev/null || true # ignore any read-only error
} }

9
contrib/upldrel/Dockerfile Normal file
View File

@ -0,0 +1,9 @@
FROM quay.io/libpod/imgts:latest
RUN yum -y update && \
yum -y install unzip && \
rpm -V unzip && \
yum clean all
COPY /contrib/upldrel/entrypoint.sh /usr/local/bin/entrypoint.sh
RUN chmod 755 /usr/local/bin/entrypoint.sh

9
contrib/upldrel/README.md Normal file
View File

@ -0,0 +1,9 @@
![PODMAN logo](../../logo/podman-logo-source.svg)
A container image for canonical-naming and uploading of
libpod and remote-client archives. Only intended to ever
be used by CI/CD, and depends heavily on an embedded
`release.txt` file produced by `make`.
Build script: [../cirrus/build_release.sh](../cirrus/build_release.sh)
Upload script: [../cirrus/upload_release_archive.sh](../cirrus/upload_release_archive.sh)

62
contrib/upldrel/entrypoint.sh Executable file
View File

@ -0,0 +1,62 @@
#!/bin/bash
set -e
source /usr/local/bin/lib_entrypoint.sh
req_env_var GCPJSON_FILEPATH GCPNAME GCPPROJECT REL_ARC_FILEPATH PR_OR_BRANCH BUCKET
[[ -r "$REL_ARC_FILEPATH" ]] || \
die 2 ERROR Cannot read release archive file: "$REL_ARC_FILEPATH"
[[ -r "$GCPJSON_FILEPATH" ]] || \
die 3 ERROR Cannot read GCP credentials file: "$GCPJSON_FILEPATH"
cd $TMPDIR
echo "Attempting to extract release.txt from tar or zip $REL_ARC_FILEPATH"
unset SFX
if tar xzf "$REL_ARC_FILEPATH" "./release.txt"
then
echo "It's a tarball"
SFX="tar.gz"
elif unzip "$REL_ARC_FILEPATH" release.txt
then
echo "It's a zip"
SFX="zip"
else
die 5 ERROR Could not extract release.txt from $REL_ARC_FILEPATH
fi
echo "Parsing release.txt contents"
RELEASETXT=$(<release.txt)
cd -
[[ -n "$RELEASETXT" ]] || \
die 3 ERROR Could not obtain metadata from release.txt in $REL_ARC_FILEPATH
RELEASE_INFO=$(echo "$RELEASETXT" | grep -m 1 'X-RELEASE-INFO:' | sed -r -e 's/X-RELEASE-INFO:\s*(.+)/\1/')
if [[ "$?" -ne "0" ]] || [[ -z "$RELEASE_INFO" ]]
then
die 4 ERROR Metadata is empty or invalid: '$RELEASETXT'
fi
# e.g. libpod v1.3.1-166-g60df124e fedora 29 amd64
# or libpod v1.3.1-166-g60df124e amd64
FIELDS="RELEASE_BASENAME RELEASE_VERSION RELEASE_DIST RELEASE_DIST_VER RELEASE_ARCH"
read $FIELDS <<< $RELEASE_INFO
for f in $FIELDS
do
[[ -n "${!f}" ]] || \
die 5 ERROR Expecting $f to be non-empty in metadata: '$RELEASE_INFO'
done
gcloud_init "$GCPJSON_FILEPATH"
# Drop version number to enable "latest" representation
# (version available w/in zip-file comment)
RELEASE_ARCHIVE_NAME="${RELEASE_BASENAME}-${PR_OR_BRANCH}-${RELEASE_DIST}-${RELEASE_DIST_VER}-${RELEASE_ARCH}.${SFX}"
echo "Uploading archive as $RELEASE_ARCHIVE_NAME"
gsutil cp "$REL_ARC_FILEPATH" "gs://$BUCKET/$RELEASE_ARCHIVE_NAME"
echo "Release now available at:"
echo " https://storage.cloud.google.com/$BUCKET/$RELEASE_ARCHIVE_NAME"

54
hack/get_release_info.sh Executable file
View File

@ -0,0 +1,54 @@
#!/bin/bash
# This script produces various bits of metadata needed by Makefile. Using
# a script allows uniform behavior across multiple environments and
# distributions. The script expects a single argument, as reflected below.
set -e
cd "${GOSRC:-$(dirname $0)/../}"
valid_args() {
REGEX='^\s+[[:upper:]]+\*[)]'
egrep --text --no-filename --group-separator=' ' --only-matching "$REGEX" "$0" | \
cut -d '*' -f 1
}
unset OUTPUT
case "$1" in
# Wild-card suffix needed by valid_args() e.g. possible bad grep of "$(echo $FOO)"
VERSION*)
OUTPUT="${CIRRUS_TAG:-$(git fetch --tags && git describe HEAD 2> /dev/null)}"
;;
NUMBER*)
OUTPUT="$($0 VERSION | sed 's/-.*//')"
;;
DIST_VER*)
OUTPUT="$(source /etc/os-release; echo $VERSION_ID | cut -d '.' -f 1)"
;;
DIST*)
OUTPUT="$(source /etc/os-release; echo $ID)"
;;
ARCH*)
OUTPUT="${GOARCH:-$(go env GOARCH 2> /dev/null)}"
;;
BASENAME*)
OUTPUT="${CIRRUS_REPO_NAME:-$(basename $(git rev-parse --show-toplevel))}"
;;
REMOTENAME*)
OUTPUT="$($0 BASENAME)-remote"
;;
*)
echo "Error, unknown/unsupported argument '$1', valid arguments:"
valid_args
exit 1
;;
esac
if [[ -n "$OUTPUT" ]]
then
echo -n "$OUTPUT"
else
echo "Error, empty output for info: '$1'" > /dev/stderr
exit 2
fi