Cirrus: Execute Windows podman-machine e2e tests

Also, de-duplicate power-shell variables and functions as they're
beginning to sprawl.  This does not completely address all duplicates,
mainly those involved in the podman-machine CI workflow.  So,
nothing under `contrib/win-installer` has been touched.

[NO NEW TESTS NEEDED]

Signed-off-by: Chris Evich <cevich@redhat.com>

.cirrus.yml

@@ -33,7 +33,7 @@ env:
     DEBIAN_NAME: "debian-13"
 
     # Image identifiers
-    IMAGE_SUFFIX: "c20230928t004553z-f39f38d13"
+    IMAGE_SUFFIX: "c20231004t194547z-f39f38d13"
 
     # EC2 images
     FEDORA_AMI: "fedora-aws-${IMAGE_SUFFIX}"
@@ -52,7 +52,7 @@ env:
     #### N/B: Required ALL of these are set for every single task.
     ####
     TEST_FLAVOR:             # int, sys, ext_svc, validate, automation, etc.
-    TEST_ENVIRON: host       # 'host', or 'container'
+    TEST_ENVIRON: host       # 'host', or 'container'.
     PODBIN_NAME: podman      # 'podman' or 'remote'
     PRIV_NAME: root          # 'root' or 'rootless'
     DISTRO_NV:               # any {PRIOR_,}{FEDORA,DEBIAN}_NAME value
@@ -367,6 +367,8 @@ alt_build_task:
       - env:
             ALT_NAME: 'Build Each Commit'
       - env:
+            # TODO: Replace with task using `winmake` to build
+            # binary and archive installation zip file.
             ALT_NAME: 'Windows Cross'
       - env:
             ALT_NAME: 'Alt Arch. x86 Cross'
@@ -397,19 +399,50 @@ win_installer_task:
       $CIRRUS_BASE_BRANCH !=~ 'v[0-9\.]+-rhel'
     depends_on:
       - alt_build
-    ec2_instance:
+    ec2_instance: &windows
         image: "${WINDOWS_AMI}"
         type: m5.large
         region: us-east-1
         platform: windows
-    env:
-        PATH: "${PATH};C:\\ProgramData\\chocolatey\\bin"
+    env: &winenv
+        CIRRUS_WORKING_DIR: &wincwd "${LOCALAPPDATA}\\cirrus-ci-build"
         CIRRUS_SHELL: powershell
+        PATH: "${PATH};C:\\ProgramData\\chocolatey\\bin"
+        CI_DESIRED_DATABASE: ""
+        DISTRO_NV: "windows"
+        PRIV_NAME: "rootless"
         # Fake version, we are only testing the installer functions, so version doesn't matter
         WIN_INST_VER: 9.9.9
-        CIRRUS_WORKING_DIR: "${LOCALAPPDATA}\\cirrus-ci-build"
-    install_script: '.\contrib\cirrus\win-installer-install.ps1'
-    main_script: '.\contrib\cirrus\win-installer-main.ps1'
+    # It's HIGHLY desireable to use the same binary throughout CI.  Otherwise, if
+    # there's a toolchain or build-environment specific problem, it can be incredibly
+    # difficult (and non-obvious) to debug.
+    clone_script: &winclone |
+        $ErrorActionPreference = 'Stop'
+        $ProgressPreference = 'SilentlyContinue'
+        New-Item -ItemType Directory -Force -Path "$ENV:CIRRUS_WORKING_DIR"
+        Set-Location "$ENV:CIRRUS_WORKING_DIR"
+        $uri = "${ENV:ART_URL}/Windows Cross/repo/repo.tbz"
+        Write-Host "Downloading $uri"
+        For($i = 0;;) {
+            Try {
+                Invoke-WebRequest -UseBasicParsing -ErrorAction Stop -OutFile "repo.tbz2" `
+                  -Uri "$uri"
+                Break
+            } Catch {
+                if (++$i -gt 6) {
+                    throw $_.Exception
+                }
+                Write-Host "Download failed - retrying:" $_.Exception.Response.StatusCode
+                Start-Sleep -Seconds 10
+            }
+        }
+        arc unarchive repo.tbz2 .\
+        if ($LASTEXITCODE -ne 0) {
+            throw "Unarchive repo.tbz2 failed"
+            Exit 1
+        }
+        Get-ChildItem -Path .\repo
+    main_script: ".\\repo\\contrib\\cirrus\\win-installer-main.ps1"
 
 
 # Confirm building the remote client, natively on a Mac OS-X VM.
@@ -585,32 +618,6 @@ compose_test_task:
     always: *logs_artifacts
 
 
-# Execute the podman integration tests on all primary platforms and release
-windows_smoke_test_task:
-    name: "Windows Smoke Test"
-    alias: windows_smoke_test
-    # Only run for non-docs/copr PRs and non-release branch builds
-    # and never for tags.  Docs: ./contrib/cirrus/CIModes.md
-    only_if: >-
-        $CIRRUS_TAG == '' &&
-        $CIRRUS_CHANGE_TITLE !=~ '.*CI:DOCS.*' &&
-        $CIRRUS_BRANCH !=~ 'v[0-9\.]+-rhel' &&
-        $CIRRUS_BASE_BRANCH !=~ 'v[0-9\.]+-rhel'
-    depends_on:
-      - alt_build
-    ec2_instance:
-        image: "${WINDOWS_AMI}"
-        type: m5zn.metal
-        region: us-east-1
-        platform: windows
-    env:
-        PATH: "${PATH};C:\\ProgramData\\chocolatey\\bin"
-        CIRRUS_SHELL: powershell
-        # Fake version, we are only testing the installer functions, so version doesn't matter
-        CIRRUS_WORKING_DIR: "${LOCALAPPDATA}\\Temp\\cirrus-ci-build"
-    main_script: 'contrib/cirrus/win-podman-machine-main.ps1'
-
-
 # versions, as root, without involving the podman-remote client.
 local_integration_test_task: &local_integration_test_task
     # Integration-test task name convention:
@@ -723,7 +730,7 @@ podman_machine_task:
         region: us-east-1
     env:
       EC2_INST_TYPE: "m5zn.metal"  # Bare-metal instance is required
-      TEST_FLAVOR: "machine"
+      TEST_FLAVOR: "machine-linux"
       PRIV_NAME: "rootless"  # intended use-case
       DISTRO_NV: "${FEDORA_NAME}"
       VM_IMAGE_NAME: "${FEDORA_AMI}"
@@ -748,7 +755,7 @@ podman_machine_aarch64_task:
     ec2_instance:
         <<: *standard_build_ec2_aarch64
     env:
-        TEST_FLAVOR: "machine"
+        TEST_FLAVOR: "machine-linux"
         EC2_INST_TYPE: c6g.metal
         PRIV_NAME: "rootless"  # intended use-case
         DISTRO_NV: "${FEDORA_AARCH64_NAME}"
@@ -760,6 +767,41 @@ podman_machine_aarch64_task:
     always: *int_logs_artifacts
 
 
+podman_machine_windows_task:
+    name: *std_name_fmt
+    alias: podman_machine_windows
+    # TODO: These tests are new and mostly fail.  Disable all failures impacting overall CI status
+    # until the tests, scripts, and environment stabalize.
+    allow_failures: $CI == $CI
+    # Only run for non-docs/copr PRs and non-release branch builds
+    # and never for tags.  Docs: ./contrib/cirrus/CIModes.md
+    only_if: >-
+        $CIRRUS_TAG == '' &&
+        $CIRRUS_CHANGE_TITLE !=~ '.*CI:DOCS.*' &&
+        $CIRRUS_BRANCH !=~ 'v[0-9\.]+-rhel' &&
+        $CIRRUS_BASE_BRANCH !=~ 'v[0-9\.]+-rhel'
+    depends_on:
+      - alt_build
+      - build
+      - win_installer
+      - local_integration_test
+      - remote_integration_test
+      - container_integration_test
+      - rootless_integration_test
+    ec2_instance:
+        <<: *windows
+        type: m5zn.metal
+        platform: windows
+    env: *winenv
+    matrix:
+      - env:
+          TEST_FLAVOR: "machine-wsl"
+      - env:
+          TEST_FLAVOR: "machine-hyperv"
+    clone_script: *winclone
+    main_script: ".\\repo\\contrib\\cirrus\\win-podman-machine-main.ps1"
+
+
 # Always run subsequent to integration tests.  While parallelism is lost
 # with runtime, debugging system-test failures can be more challenging
 # for some golang developers.  Otherwise the following tasks run across
@@ -994,7 +1036,6 @@ success_task:
         - osx_alt_build
         - freebsd_alt_build
         - win_installer
-        - windows_smoke_test
         - docker-py_test
         - unit_test
         - apiv2_test
@@ -1005,6 +1046,7 @@ success_task:
         - rootless_integration_test
         - podman_machine
         - podman_machine_aarch64
+        - podman_machine_windows
         - local_system_test
         - local_system_test_aarch64
         - remote_system_test

contrib/cirrus/runner.sh

@@ -418,7 +418,7 @@ dotest() {
         |& logformatter
 }
 
-_run_machine() {
+_run_machine-linux() {
     # N/B: Can't use _bail_if_test_can_be_skipped here b/c content isn't under test/
     showrun make localmachine |& logformatter
 }

contrib/cirrus/setup_environment.sh

@@ -413,7 +413,7 @@ case "$TEST_FLAVOR" in
         showrun minikube config set driver podman
         install_test_configs
         ;;
-    machine)
+    machine-linux)
         showrun dnf install -y podman-gvproxy*
         remove_packaged_podman_files
         showrun make install PREFIX=/usr ETCDIR=/etc

contrib/cirrus/win-installer-install.ps1

@@ -1,27 +0,0 @@
-# Update service is required for dotnet 3.5 (dep of wix)
-Set-Service -Name wuauserv -StartupType "Manual"
-function retryInstall {
-    param([Parameter(ValueFromRemainingArguments)] [string[]] $pkgs)
-
-    foreach ($pkg in $pkgs) {
-        for ($retries = 0; ; $retries++) {
-            if ($retries -gt 5) {
-                throw "Could not install package $pkg"
-            }
-
-            if ($pkg -match '(.[^\@]+)@(.+)') {
-                $pkg = @("--version", $Matches.2, $Matches.1)
-            }
-
-            choco install -y --allow-downgrade $pkg
-            if ($LASTEXITCODE -eq 0) {
-                break
-            }
-            Write-Output "Error installing, waiting before retry..."
-            Start-Sleep -Seconds 6
-        }
-    }
-}
-# Force mingw version 11.2 since later versions are incompatible
-# with CGO on some versions of golang
-retryInstall wixtoolset mingw@11.2 golang archiver

contrib/cirrus/win-installer-main.ps1

@@ -1,52 +1,12 @@
- # Powershell doesn't exit after
- function CheckExit {
-    if ($LASTEXITCODE -ne 0) {
-        throw "Exit code = $LASTEXITCODE"
-    }
-}
-function DownloadFile {
-    param(
-        [Parameter(Mandatory)]
-        [string]$url,
-        [Parameter(Mandatory)]
-        [string]$file,
-        [Int]$retries=5,
-        [Int]$delay=8
-    )
-    $ProgressPreference = 'SilentlyContinue';
-    Write-Host "Downloading $url to $file"
-    For($i = 0;;) {
-        Try {
-            Invoke-WebRequest -UseBasicParsing -ErrorAction Stop -Uri $url -OutFile $file
-            Break
-        } Catch {
-            if (++$i -gt $retries) {
-                throw $_.Exception
-            }
-            Write-Host "Download failed - retrying:" $_.Exception.Response.StatusCode
-            Start-Sleep -Seconds $delay
-        }
-    }
-}
-# Drop global envs which have unix paths, defaults are fine
-Remove-Item Env:\GOPATH -ErrorAction:Ignore
-Remove-Item Env:\GOSRC -ErrorAction:Ignore
-Remove-Item Env:\GOCACHE -ErrorAction:Ignore
+#!/usr/bin/env powershell
 
-# Drop large known env variables (an env > 32k will break MSI/ICE validation)
-Remove-Item Env:\CIRRUS_COMMIT_MESSAGE -ErrorAction:Ignore
-Remove-Item Env:\CIRRUS_PR_BODY -ErrorAction:Ignore
+. $PSScriptRoot\win-lib.ps1
 
-Set-Location contrib\win-installer
-
-# Download and extract alt_build win release zip
-$url = "${ENV:ART_URL}/Windows Cross/repo/repo.tbz"
-# Arc requires extension to be "tbz2"
-DownloadFile "$url" "repo.tbz2"
-arc unarchive repo.tbz2 .; CheckExit
+Set-Location "$ENV:CIRRUS_WORKING_DIR\repo\contrib\win-installer"
 
 # Build Installer
-.\build.ps1 $Env:WIN_INST_VER dev repo; CheckExit
+# Note: consumes podman-remote-release-windows_amd64.zip from repo.tbz2
+Run-Command ".\build.ps1 $Env:WIN_INST_VER dev `"$ENV:CIRRUS_WORKING_DIR\repo`""
 
 # Run the installer silently and WSL install option disabled (prevent reboots, wsl requirements)
 # We need AllowOldWin=1 for server 2019 (cirrus image), can be dropped after server 2022

contrib/cirrus/win-lib.ps1

@@ -0,0 +1,72 @@
+#!/usr/bin/env powershell
+
+# This powershell script is intended to be "dot-sourced" by other scripts.
+# It's purpose is identical to that of the `lib.sh` script for Linux environments.
+
+# Behave similar to `set -e` in bash, but ONLY for powershell commandlets!
+# For all legacy, program, and script calls use Run-Command() or Check-Exit()
+$ErrorActionPreference = 'Stop'
+
+# Any golang compilation needs to know what it's building for.
+$Env:GOOS = "windows"
+$Env:GOARCH = "amd64"
+
+# Unnecessary and intrusive.  They claim parameter/variable
+# values aren't collected, but there could be a bug leading
+# to a concern over leaking of some sensitive-value.  Stop this.
+$Env:POWERSHELL_TELEMETRY_OPTOUT = "true"
+
+# Unnecessary and potentially disruptive.  Powershell will
+# never ever be updated during automation execution.  Stop this.
+$Env:POWERSHELL_UPDATECHECK = "off"
+
+# Color in output may confuse tooling and makes logs hard to read.
+# TODO: There are probably other places where color needs to be disabled
+# in a slightly different way :(
+$Env:NO_COLOR = "true"
+
+# Items only relevant in a CI environment.
+if ($Env:CI -eq "true") {
+    # Defined by .cirrus.yml for use by all the linux tasks.
+    # Drop all global envs which have unix paths, defaults are fine.
+    Remove-Item Env:\GOPATH -ErrorAction:Ignore
+    Remove-Item Env:\GOSRC -ErrorAction:Ignore
+    Remove-Item Env:\GOCACHE -ErrorAction:Ignore
+
+    # Defined by Cirrus-CI
+    # Drop large known env variables (an env > 32k will break MSI/ICE validation)
+    Remove-Item Env:\CIRRUS_COMMIT_MESSAGE -ErrorAction:Ignore
+    Remove-Item Env:\CIRRUS_CHANGE_MESSAGE -ErrorAction:Ignore
+    Remove-Item Env:\CIRRUS_PR_BODY -ErrorAction:Ignore
+}
+
+# Non-powershell commands do not halt execution on error!  This helper
+# should be called after every critical operation to check and halt on a
+# non-zero exit code.  Be careful not to use this for powershell commandlets
+# (builtins)!  They set '$?' to "True" (failed) or "False" success so calling
+# this would mask failures.  Rely on $ErrorActionPreference = 'Stop' instead.
+function Check-Exit {
+    $result = $LASTEXITCODE  # WARNING: might not be a number!
+    if ( ($result -ne $null) -and ($result -ne 0) ) {
+        # https://learn.microsoft.com/en-us/dotnet/api/system.management.automation.callstackframe
+        $caller = (Get-PSCallStack)[1]
+        Write-Host "Exit code = '$result' from $($caller.ScriptName):$($caller.ScriptLineNumber)"
+        Exit $result
+    }
+}
+
+# Small helper to avoid needing to write 'Check-Exit' after every
+# non-powershell instruction.  It simply prints then executes the _QUOTED_
+# argument followed by Check-Exit.
+# N/B: Escape any nested quotes with back-tick ("`") characters.
+# WARNING: DO NOT use this with powershell builtins! It will not do what you expect!
+function Run-Command {
+    param (
+        [string] $command
+    )
+
+    Write-Host $command
+
+    Invoke-Expression $command
+    Check-Exit
+}

contrib/cirrus/win-podman-machine-main.ps1

@@ -1,39 +1,20 @@
-$ErrorActionPreference = 'Stop'
+#!/usr/bin/env powershell
 
-# Powershell doesn't exit after command failures
-# Note, due to a bug in cirrus that does not correctly evaluate exit
-# code, error conditions should always be thrown
-function CheckExit {
-    if ($LASTEXITCODE -ne 0) {
-        throw "Exit code failure = $LASTEXITCODE"
-    }
-}
+. $PSScriptRoot\win-lib.ps1
 
-# Drop global envs which have unix paths, defaults are fine
-Remove-Item Env:\GOPATH -ErrorAction:Ignore
-Remove-Item Env:\GOSRC -ErrorAction:Ignore
-Remove-Item Env:\GOCACHE -ErrorAction:Ignore
+Set-Location "$ENV:CIRRUS_WORKING_DIR\repo"
 
-mkdir tmp
-Set-Location tmp
-
-# Download and extract alt_build win release zip
-$url = "${ENV:ART_URL}/Windows%20Cross/repo/repo.tbz"
-Write-Output "URL: $url"
-# Arc requires extension to be "tbz2"
-curl.exe -L -o repo.tbz2 "$url"; CheckExit
-arc unarchive repo.tbz2 .; CheckExit
-Set-Location repo
-Expand-Archive -Path "podman-remote-release-windows_amd64.zip" `
-               -DestinationPath extracted
-Set-Location extracted
-$x = Get-ChildItem -Path bin -Recurse
-Set-Location $x
+Write-Host "Saving selection of CI env. vars."
+# Env. vars will not pass through win-sess-launch.ps1
+Get-ChildItem -Path "Env:\*" -include @("PATH", "Chocolatey*", "CIRRUS*", "TEST_*", "CI_*") `
+  | Export-CLIXML "$ENV:TEMP\envars.xml"
 
 # Recent versions of WSL are packaged as a Windows store app running in
 # an appX container, which is incompatible with non-interactive
 # session 0 execution (where the cirrus agent runs).
 # Run verification under an interactive session instead.
-powershell.exe -File "$PSScriptRoot\wsl-env-launch.ps1" `
-                     "$PSScriptRoot\win-podman-machine-verify.ps1"
-CheckExit
+Write-Host "Spawning new session to execute $PSScriptRoot\win-podman-machine-test.ps1"
+# Can't use Run-Command(), would need overly-complex nested quoting
+powershell.exe -File "$PSScriptRoot\win-sess-launch.ps1" `
+                     "$PSScriptRoot\win-podman-machine-test.ps1"
+Check-Exit

contrib/cirrus/win-podman-machine-test.ps1

@@ -0,0 +1,33 @@
+#!/usr/bin/env powershell
+
+. $PSScriptRoot\win-lib.ps1
+
+Write-Host "Recovering env. vars."
+Import-CLIXML "$ENV:TEMP\envars.xml" | % {
+    Write-Host "    $($_.Name) = $($_.Value)"
+    Set-Item "Env:$($_.Name)" "$($_.Value)"
+}
+
+if ($Env:TEST_FLAVOR -eq "machine-wsl") {
+    # FIXME: Test-modes should be definitively set and positively asserted.
+    # Otherwise if the var. goes out-of-scope, defaults change, or definition
+    # fails: Suddenly assumed behavior != actual behaviorr, esp. if/when only
+    # quickly glancing at a green status check-mark.
+    $Env:CONTAINERS_MACHINE_PROVIDER = ""  # IMPLIES WSL
+} elseif ($Env:TEST_FLAVOR -eq "machine-hyperv") {
+    $Env:CONTAINERS_MACHINE_PROVIDER = "hyperv"
+} else {
+    Write-Host "Unsupported value for `$TEST_FLAVOR '$Env:TEST_FLAVOR'"
+    Exit 1
+}
+# Make sure an observer knows the value of this critical variable (consumed by tests).
+Write-Host "    CONTAINERS_MACHINE_PROVIDER = $Env:CONTAINERS_MACHINE_PROVIDER"
+Write-Host "`n"
+
+# The repo.tbz artifact was extracted here
+Set-Location "$ENV:CIRRUS_WORKING_DIR\repo"
+# Tests hard-code this location for podman-remote binary, make sure it actually runs.
+Run-Command ".\bin\windows\podman.exe --version"
+
+Write-Host "`nRunning podman-machine e2e tests"
+Run-Command ".\winmake localmachine"

contrib/cirrus/win-podman-machine-verify.ps1

@@ -1,27 +0,0 @@
-$ErrorActionPreference = 'Stop'
-function CheckExit {
-    if ($LASTEXITCODE -ne 0) {
-        throw "Exit code failure = $LASTEXITCODE"
-    }
-}
-
-# Verify extracted podman binary
-Write-Output `n"Starting init...`n"
-.\podman machine init; CheckExit
-Write-Output "`nStarting podman machine...`n"
-.\podman machine start; CheckExit
-Write-Output "`nDumping info...`n"
-for ($i =0; $i -lt 60; $i++) {
-    .\podman info
-    if ($LASTEXITCODE -eq 0) {
-        break
-    }
-    Start-Sleep -Seconds 2
-}
-Write-Output "`nRunning container...`n"
-.\podman run ubi8-micro sh -c "exit 123"
-if ($LASTEXITCODE -ne 123) {
-    throw  "Expected 123, got $LASTEXITCODE"
-}
-Write-Host "`nMachine verification is successful!`n"
-Exit 0

winmake.ps1

@@ -1,4 +1,5 @@
-$ErrorActionPreference = 'Stop'
+
+. ./contrib/cirrus/win-lib.ps1
 
 # Targets
 function Podman-Remote{
@@ -68,26 +69,10 @@ function Git-Commit{
     return $commit
 }
 
-function Run-Command {
-    param (
-        [string] $command
-    )
-
-    Write-Host $command
-
-    Invoke-Expression $command
-    $result = $LASTEXITCODE
-    if ($result -ne 0) {
-        Write-Host "Command failed (exit: $result)"
-        Exit $result
-    }
-}
-
 # Init script
 $target = $args[0]
 
 $remotetags = "remote exclude_graphdriver_btrfs btrfs_noversion exclude_graphdriver_devicemapper containers_image_openpgp"
-$Env:GOOS = "windows"; $Env:GOARCH = "amd64"
 
 switch ($target) {
     {$_ -in '', 'podman-remote', 'podman'} {