Merge pull request #24120 from Luap99/cirrus-build-speed

cirrus: use fastvm for builds

.cirrus.yml

@@ -129,10 +129,10 @@ validate-source_task:
 build_task:
     alias: 'build'
     name: 'Build for $DISTRO_NV' # N/B: Referenced by URLencoded strings elsewhere
-    gce_instance: &standardvm
+    gce_instance: &fastvm
         image_project: libpod-218412
         zone: "us-central1-a"
-        cpu: 2
+        cpu: 4
         memory: "4Gb"
         # Required to be 200gig, do not modify - has i/o performance impact
         # according to gcloud CLI tool warning messages.
@@ -178,9 +178,9 @@ build_task:
     # Cirrus-CI is very slow uploading one file at time, and the repo contains
     # thousands of files.  Speed this up by archiving into tarball first.
     repo_prep_script: &repo_prep >-
-        tar cjf /tmp/repo.tbz -C $GOSRC . && mv /tmp/repo.tbz $GOSRC/
+        tar --zstd -cf /tmp/repo.tar.zst -C $GOSRC . && mv /tmp/repo.tar.zst $GOSRC/
     repo_artifacts: &repo_artifacts
-        path: ./repo.tbz
+        path: ./repo.tar.zst
         type: application/octet-stream
     always: &runner_stats
         runner_stats_artifacts:
@@ -210,11 +210,8 @@ build_aarch64_task:
     main_script: *main
     # Cirrus-CI is very slow uploading one file at time, and the repo contains
     # thousands of files.  Speed this up by archiving into tarball first.
-    repo_prep_script: &repo_prep_aarch64 >-
+    repo_prep_script: *repo_prep
-        tar cjf /tmp/repo.tbz -C $GOSRC . && mv /tmp/repo.tbz $GOSRC/
+    repo_artifacts: *repo_artifacts
-    repo_artifacts: &repo_artifacts_aarch64
-        path: ./repo.tbz
-        type: application/octet-stream
     always: *runner_stats
 
 
@@ -232,7 +229,7 @@ alt_build_task:
     env:
         <<: *stdenvars
         TEST_FLAVOR: "altbuild"
-    gce_instance: *standardvm
+    gce_instance: *fastvm
     matrix:
       - env:
             ALT_NAME: 'Build Each Commit'
@@ -250,11 +247,11 @@ alt_build_task:
             ALT_NAME: 'Alt Arch. MIPS64 Cross'
       - env:
             ALT_NAME: 'Alt Arch. Other Cross'
-    # This task cannot make use of the shared repo.tbz artifact.
+    # This task cannot make use of the shared repo.tar.zst artifact.
     clone_script: *full_clone
     setup_script: *setup
     main_script: *main
-    # Produce a new repo.tbz artifact for consumption by 'artifacts' task.
+    # Produce a new repo.tar.zst artifact for consumption by 'artifacts' task.
     repo_prep_script: *repo_prep
     repo_artifacts: *repo_artifacts
     always: *runner_stats
@@ -301,7 +298,7 @@ osx_alt_build_task:
     # The Mac tests rely this Podman binary to run, and the CI Mac is ARM-based
     build_arm64_script:
         - make podman-remote-release-darwin_arm64.zip
-    # Produce a new repo.tbz artifact for consumption by dependent tasks.
+    # Produce a new repo.tar.zst artifact for consumption by dependent tasks.
     repo_prep_script: *repo_prep
     repo_artifacts: *repo_artifacts
     # This host is/was shared with potentially many other CI tasks.
@@ -329,11 +326,11 @@ freebsd_alt_build_task:
     freebsd_instance:
         image_family: freebsd-13-3
     setup_script:
-        - pkg install -y gpgme bash go-md2man gmake gsed gnugrep go pkgconf
+        - pkg install -y gpgme bash go-md2man gmake gsed gnugrep go pkgconf zstd
     build_amd64_script:
         - gmake podman-release
-    # This task cannot make use of the shared repo.tbz artifact and must
+    # This task cannot make use of the shared repo.tar.zst artifact and must
-    # produce a new repo.tbz artifact for consumption by 'artifacts' task.
+    # produce a new repo.tar.zst artifact for consumption by 'artifacts' task.
     repo_prep_script: *repo_prep
     repo_artifacts: *repo_artifacts
 
@@ -380,16 +377,18 @@ bindings_task:
         (changesInclude('**/*.go', '**/*.c', '**/*.h') && !changesIncludeOnly('test/**', 'pkg/machine/e2e/**'))
     depends_on: &build
         - build_success
-    gce_instance: *standardvm
+    gce_instance: &standardvm
+      <<: *fastvm
+      cpu: 2
     env:
         <<: *stdenvars
         TEST_FLAVOR: bindings
     # N/B: This script depends on ${DISTRO_NV} being defined for the task.
     clone_script: &get_gosrc |
         cd /tmp
-        echo "$ARTCURL/Build%20for%20${DISTRO_NV}/repo/repo.tbz"
+        echo "$ARTCURL/Build%20for%20${DISTRO_NV}/repo/repo.tar.zst"
-        time $ARTCURL/Build%20for%20${DISTRO_NV}/repo/repo.tbz
+        time $ARTCURL/Build%20for%20${DISTRO_NV}/repo/repo.tar.zst
-        time tar xjf /tmp/repo.tbz -C $GOSRC
+        time tar -xf /tmp/repo.tar.zst -C $GOSRC
     setup_script: *setup
     main_script: *main
     always: &logs_artifacts
@@ -466,11 +465,11 @@ win_installer_task:
         $ProgressPreference = 'SilentlyContinue'
         New-Item -ItemType Directory -Force -Path "$ENV:CIRRUS_WORKING_DIR"
         Set-Location "$ENV:CIRRUS_WORKING_DIR"
-        $uri = "${ENV:ART_URL}/Windows Cross/repo/repo.tbz"
+        $uri = "${ENV:ART_URL}/Windows Cross/repo/repo.tar.zst"
         Write-Host "Downloading $uri"
         For($i = 0;;) {
             Try {
-                Invoke-WebRequest -UseBasicParsing -ErrorAction Stop -OutFile "repo.tbz2" `
+                Invoke-WebRequest -UseBasicParsing -ErrorAction Stop -OutFile "repo.tar.zst" `
                   -Uri "$uri"
                 Break
             } Catch {
@@ -481,11 +480,19 @@ win_installer_task:
                 Start-Sleep -Seconds 10
             }
         }
-        arc unarchive repo.tbz2 .\
+        Write-Host "zstd -d repo.tar.zst"
+        zstd -d repo.tar.zst
         if ($LASTEXITCODE -ne 0) {
-            throw "Unarchive repo.tbz2 failed"
+            throw "Extract repo.tar.zst failed"
             Exit 1
         }
+        Write-Host "arc unarchive repo.tar .\"
+        arc unarchive repo.tar .\repo
+        if ($LASTEXITCODE -ne 0) {
+            throw "Unarchive repo.tar failed"
+            Exit 1
+        }
+        Get-ChildItem -Path .
         Get-ChildItem -Path .\repo
     main_script: ".\\repo\\contrib\\cirrus\\win-installer-main.ps1"
 
@@ -626,9 +633,7 @@ local_integration_test_task: &local_integration_test_task
     matrix: *platform_axis
     # integration tests scale well with cpu as they are parallelized
     # so we give these tests 4 cores to make them faster
-    gce_instance: &fastvm
+    gce_instance: *fastvm
-      <<: *standardvm
-      cpu: 4
     env:
         TEST_FLAVOR: int
     clone_script: *get_gosrc
@@ -734,9 +739,9 @@ podman_machine_aarch64_task:
         VM_IMAGE_NAME: "${FEDORA_AARCH64_AMI}"
     clone_script: &get_gosrc_aarch64 |
         cd /tmp
-        echo "$ARTCURL/build_aarch64/repo/repo.tbz"
+        echo "$ARTCURL/build_aarch64/repo/repo.tar.zst"
-        time $ARTCURL/build_aarch64/repo/repo.tbz
+        time $ARTCURL/build_aarch64/repo/repo.tar.zst
-        time tar xjf /tmp/repo.tbz -C $GOSRC
+        time tar -xf /tmp/repo.tar.zst -C $GOSRC
     setup_script: *setup
     main_script: *main
     always: *int_logs_artifacts
@@ -798,8 +803,8 @@ podman_machine_mac_task:
     clone_script:  # artifacts from osx_alt_build_task
         - mkdir -p $CIRRUS_WORKING_DIR
         - cd $CIRRUS_WORKING_DIR
-        - $ARTCURL/Build%20for%20MacOS%20amd64%2Barm64/repo/repo.tbz
+        - $ARTCURL/Build%20for%20MacOS%20amd64%2Barm64/repo/repo.tar.zst
-        - tar xjf repo.tbz
+        - tar -xf repo.tar.zst
     # This host is/was shared with potentially many other CI tasks.
     # The previous task may have been canceled or aborted.
     prep_script: *mac_cleanup
@@ -1108,20 +1113,20 @@ artifacts_task:
     fedora_binaries_script:
         - mkdir -p /tmp/fed
         - cd /tmp/fed
-        - $ARTCURL/Build%20for%20${FEDORA_NAME}/repo/repo.tbz
+        - $ARTCURL/Build%20for%20${FEDORA_NAME}/repo/repo.tar.zst
-        - tar xjf repo.tbz
+        - tar -xf repo.tar.zst
         - cp ./bin/* $CIRRUS_WORKING_DIR/
     win_binaries_script:
         - mkdir -p /tmp/win
         - cd /tmp/win
-        - $ARTCURL/Windows%20Cross/repo/repo.tbz
+        - $ARTCURL/Windows%20Cross/repo/repo.tar.zst
-        - tar xjf repo.tbz
+        - tar -xf repo.tar.zst
         - mv ./podman-remote*.zip $CIRRUS_WORKING_DIR/
     osx_binaries_script:
         - mkdir -p /tmp/osx
         - cd /tmp/osx
-        - $ARTCURL/Build%20for%20MacOS%20amd64%2Barm64/repo/repo.tbz
+        - $ARTCURL/Build%20for%20MacOS%20amd64%2Barm64/repo/repo.tar.zst
-        - tar xjf repo.tbz
+        - tar -xf repo.tar.zst
         - mv ./podman-remote-release-darwin_*.zip $CIRRUS_WORKING_DIR/
         - mv ./contrib/pkginstaller/out/podman-installer-macos-*.pkg $CIRRUS_WORKING_DIR/
     always:

Makefile

@@ -500,6 +500,14 @@ local-cross: $(CROSS_BUILD_TARGETS) ## Cross compile podman binary for multiple
 .PHONY: cross
 cross: local-cross
 
+# Simple target to check that we can build all binaries for another arch,
+# the resulting binaries are not meant to be usable this is just for
+# testing if it builds, it depends on the caller to set GOOS/GOARCH.
+.PHONY: cross-binaries
+cross-binaries:
+	$(MAKE) CGO_ENABLED=0 GOOS=$(GOOS) GOARCH=$(GOARCH) \
+		BUILDTAGS="$(BUILDTAGS_CROSS)" clean-binaries binaries
+
 .PHONY: completions
 completions: podman podman-remote
 	# key = shell, value = completion filename
@@ -590,10 +598,18 @@ podman-remote-%-docs: podman-remote
 		$(if $(findstring windows,$*),docs/source/markdown,docs/build/man)
 
 .PHONY: man-page-check
-man-page-check: bin/podman docs
+man-page-check: man-page-checker xref-helpmsgs-manpages xref-quadlet-docs xref-quadlet-docs
+
+man-page-checker: bin/podman docs
 	hack/man-page-checker
+
+xref-helpmsgs-manpages: bin/podman docs
 	hack/xref-helpmsgs-manpages
+
+man-page-table-check: docs
 	hack/man-page-table-check
+
+xref-quadlet-docs: docs
 	hack/xref-quadlet-docs
 
 .PHONY: swagger-check

contrib/cirrus/postbuild.sh

@@ -30,13 +30,13 @@ source $AUTOMATION_LIB_PATH/common_lib.sh
 # shellcheck disable=SC2154
 cd $CIRRUS_WORKING_DIR
 
+# Note, make completions and make vendor will already be run in _run_build()
+# so do not run them again for no reason. This just makes CI slower.
+SUGGESTION="run 'make vendor' or 'make completions' and commit all changes" ./hack/tree_status.sh
+
 showrun make .install.goimports
-showrun make vendor
-SUGGESTION="run 'make vendor' and commit all changes" ./hack/tree_status.sh
 showrun make generate-bindings
 SUGGESTION="run 'make generate-bindings' and commit all changes" ./hack/tree_status.sh
-showrun make completions
-SUGGESTION="run 'make completions' and commit all changes" ./hack/tree_status.sh
 
 # Defined in Cirrus-CI config.
 # shellcheck disable=SC2154

contrib/cirrus/runner.sh

@@ -210,19 +210,17 @@ eof
 }
 
 function _run_build() {
-    local vb_target
+    # Ensure always start from clean-slate with all vendor modules downloaded
+    showrun make clean
+    showrun make vendor
+    showrun make -j $(nproc) --output-sync=target podman-release  # includes podman, podman-remote, and docs
 
     # There's no reason to validate-binaries across multiple linux platforms
     # shellcheck disable=SC2154
     if [[ "$DISTRO_NV" =~ $FEDORA_NAME ]]; then
-        vb_target=validate-binaries
+        showrun make -j $(nproc) --output-sync=target validate-binaries
     fi
 
-    # Ensure always start from clean-slate with all vendor modules downloaded
-    showrun make clean
-    showrun make vendor
-    showrun make podman-release $vb_target # includes podman, podman-remote, and docs
-
     # Last-minute confirmation that we're testing the desired runtime.
     # This Can't Possibly Fail™ in regular CI; only when updating VMs.
     # $CI_DESIRED_RUNTIME must be defined in .cirrus.yml.
@@ -312,7 +310,7 @@ function _run_altbuild() {
 function _build_altbuild_archs() {
     for arch in "$@"; do
         msg "Building release archive for $arch"
-        showrun make podman-release-${arch}.tar.gz GOARCH=$arch
+        showrun make cross-binaries GOARCH=$arch
     done
 }
 

contrib/cirrus/win-installer-main.ps1

@@ -15,7 +15,7 @@ if ($Env:CI -eq "true") {
 Push-Location $WIN_INST_FOLDER
 
 # Build Installer
-# Note: consumes podman-remote-release-windows_amd64.zip from repo.tbz2
+# Note: consumes podman-remote-release-windows_amd64.zip from repo.tar.zst
 Run-Command ".\build.ps1 $Env:WIN_INST_VER dev `"$RELEASE_DIR`""
 
 Pop-Location

contrib/cirrus/win-podman-machine-test.ps1

@@ -24,7 +24,7 @@ if ($Env:TEST_FLAVOR -eq "machine-wsl") {
 Write-Host "    CONTAINERS_MACHINE_PROVIDER = $Env:CONTAINERS_MACHINE_PROVIDER"
 Write-Host "`n"
 
-# The repo.tbz artifact was extracted here
+# The repo.tar.zst artifact was extracted here
 Set-Location "$ENV:CIRRUS_WORKING_DIR\repo"
 # Tests hard-code this location for podman-remote binary, make sure it actually runs.
 Run-Command ".\bin\windows\podman.exe --version"