zstd now default compression for podman machine

given that we are moving to building our own machine images, we have
decided to use zstd compression as it is superior in speed to the
alternatives.  as such, this pr adds zstd to our machine code; and also
has to account for dealing with sparseness on darwin; which the default
zstd golang library does not.

[NO NEW TESTS NEEDED]

Signed-off-by: Brent Baude <bbaude@redhat.com>

go.mod

@@ -41,6 +41,7 @@ require (
 	github.com/hashicorp/go-multierror v1.1.1
 	github.com/hugelgupf/p9 v0.3.1-0.20230822151754-54f5c5530921
 	github.com/json-iterator/go v1.1.12
+	github.com/klauspost/compress v1.17.5
 	github.com/linuxkit/virtsock v0.0.0-20220523201153-1a23e78aa7a2
 	github.com/mattn/go-shellwords v1.0.12
 	github.com/mattn/go-sqlite3 v1.14.21
@@ -149,7 +150,6 @@ require (
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jinzhu/copier v0.4.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
-	github.com/klauspost/compress v1.17.5 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.6 // indirect
 	github.com/klauspost/pgzip v1.2.6 // indirect
 	github.com/kr/fs v0.1.0 // indirect

pkg/machine/compression/compression_test.go

@@ -33,11 +33,11 @@ func Test_compressionFromFile(t *testing.T) {
 			want: Bz2,
 		},
 		{
-			name: "default is xz",
+			name: "default is zstd",
 			args: args{
 				path: "/tmp/foo",
 			},
-			want: Xz,
+			want: Zstd,
 		},
 	}
 	for _, tt := range tests {
@@ -76,9 +76,9 @@ func TestImageCompression_String(t *testing.T) {
 			want: "zip",
 		},
 		{
-			name: "xz is default",
+			name: "zstd is default",
 			c:    99,
-			want: "xz",
+			want: "zst",
 		},
 	}
 	for _, tt := range tests {

pkg/machine/compression/config.go

@@ -9,6 +9,7 @@ const (
 	Zip
 	Gz
 	Bz2
+	Zstd
 )
 
 func KindFromFile(path string) ImageCompression {
@@ -19,9 +20,11 @@ func KindFromFile(path string) ImageCompression {
 		return Gz
 	case strings.HasSuffix(path, Zip.String()):
 		return Zip
-	}
+	case strings.HasSuffix(path, Xz.String()):
 		return Xz
 	}
+	return Zstd
+}
 
 func (c ImageCompression) String() string {
 	switch c {
@@ -31,6 +34,8 @@ func (c ImageCompression) String() string {
 		return "zip"
 	case Bz2:
 		return "bz2"
-	}
+	case Xz:
 		return "xz"
 	}
+	return "zst"
+}

pkg/machine/compression/decompress.go

@@ -18,6 +18,7 @@ import (
 	"github.com/containers/podman/v5/utils"
 	"github.com/containers/storage/pkg/archive"
 	crcOs "github.com/crc-org/crc/v2/pkg/os"
+	"github.com/klauspost/compress/zstd"
 	"github.com/sirupsen/logrus"
 	"github.com/ulikunitz/xz"
 )
@@ -59,12 +60,22 @@ func Decompress(localPath *define.VMFile, uncompressedPath string) error {
 		if err != nil {
 			return err
 		}
+		// darwin really struggles with sparse files. being diligent here
 		fmt.Printf("Copying uncompressed file %q to %q/n", localPath.GetPath(), dstFile.Name())
+
+		// Keeping CRC implementation for now, but ideally this could be pruned and
+		// sparsewriter could be used.  in that case, this area needs rework or
+		// sparsewriter be made to honor the *file interface
 		_, err = crcOs.CopySparse(uncompressedFileWriter, dstFile)
 		return err
 	case archive.Gzip:
 		if runtime.GOOS == "darwin" {
-			return decompressGzWithSparse(prefix, localPath, uncompressedPath)
+			return decompressGzWithSparse(prefix, localPath, uncompressedFileWriter)
+		}
+		fallthrough
+	case archive.Zstd:
+		if runtime.GOOS == "darwin" {
+			return decompressZstdWithSparse(prefix, localPath, uncompressedFileWriter)
 		}
 		fallthrough
 	default:
@@ -225,22 +236,31 @@ func decompressZip(prefix string, src string, output io.WriteCloser) error {
 	return err
 }
 
-func decompressGzWithSparse(prefix string, compressedPath *define.VMFile, uncompressedPath string) error {
+func decompressWithSparse(prefix string, compressedReader io.Reader, uncompressedFile *os.File) error {
-	stat, err := os.Stat(compressedPath.GetPath())
+	dstFile := NewSparseWriter(uncompressedFile)
-	if err != nil {
-		return err
-	}
-
-	dstFile, err := os.OpenFile(uncompressedPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, stat.Mode())
-	if err != nil {
-		return err
-	}
 	defer func() {
 		if err := dstFile.Close(); err != nil {
-			logrus.Errorf("unable to close uncompressed file %s: %q", uncompressedPath, err)
+			logrus.Errorf("unable to close uncompressed file %s: %q", uncompressedFile.Name(), err)
 		}
 	}()
 
+	// TODO remove the following line when progress bars work
+	_ = prefix
+	// p, bar := utils.ProgressBar(prefix, stat.Size(), prefix+": done")
+	// proxyReader := bar.ProxyReader(f)
+	// defer func() {
+	// 	if err := proxyReader.Close(); err != nil {
+	// 		logrus.Error(err)
+	// 	}
+	// }()
+
+	// p.Wait()
+	_, err := io.Copy(dstFile, compressedReader)
+	return err
+}
+
+func decompressGzWithSparse(prefix string, compressedPath *define.VMFile, uncompressedFileWriter *os.File) error {
+	logrus.Debugf("decompressing %s", compressedPath.GetPath())
 	f, err := os.Open(compressedPath.GetPath())
 	if err != nil {
 		return err
@@ -260,20 +280,34 @@ func decompressGzWithSparse(prefix string, compressedPath *define.VMFile, uncomp
 			logrus.Errorf("unable to close gzreader: %q", err)
 		}
 	}()
-
+	// This way we get something to look at in debug mode
-	// TODO remove the following line when progress bars work
+	defer func() {
-	_ = prefix
-	// p, bar := utils.ProgressBar(prefix, stat.Size(), prefix+": done")
-	// proxyReader := bar.ProxyReader(f)
-	// defer func() {
-	// 	if err := proxyReader.Close(); err != nil {
-	// 		logrus.Error(err)
-	// 	}
-	// }()
-
-	logrus.Debugf("decompressing %s", compressedPath.GetPath())
-	_, err = crcOs.CopySparse(dstFile, gzReader)
 		logrus.Debug("decompression complete")
-	// p.Wait()
+	}()
+	return decompressWithSparse(prefix, gzReader, uncompressedFileWriter)
+}
+
+func decompressZstdWithSparse(prefix string, compressedPath *define.VMFile, uncompressedFileWriter *os.File) error {
+	logrus.Debugf("decompressing %s", compressedPath.GetPath())
+	f, err := os.Open(compressedPath.GetPath())
+	if err != nil {
 		return err
 	}
+	defer func() {
+		if err := f.Close(); err != nil {
+			logrus.Errorf("unable to close on compressed file %s: %q", compressedPath.GetPath(), err)
+		}
+	}()
+
+	zstdReader, err := zstd.NewReader(f)
+	if err != nil {
+		return err
+	}
+	defer zstdReader.Close()
+
+	// This way we get something to look at in debug mode
+	defer func() {
+		logrus.Debug("decompression complete")
+	}()
+	return decompressWithSparse(prefix, zstdReader, uncompressedFileWriter)
+}

pkg/machine/define/image_format.go

@@ -1,5 +1,7 @@
 package define
 
+import "fmt"
+
 type ImageFormat int64
 
 const (
@@ -22,13 +24,9 @@ func (imf ImageFormat) Kind() string {
 }
 
 func (imf ImageFormat) KindWithCompression() string {
-	switch imf {
+	// Tar uses xz; all others use zstd
-	case Vhdx:
+	if imf == Tar {
-		return "vhdx.zip"
-	case Tar:
 		return "tar.xz"
-	case Raw:
-		return "raw.gz"
 	}
-	return "qcow2.xz"
+	return fmt.Sprintf("%s.zst", imf.Kind())
 }

pkg/machine/define/image_format_test.go

@@ -45,19 +45,19 @@ func TestImageFormat_KindWithCompression(t *testing.T) {
 		want string
 	}{
 		{
-			name: "vhdx.zip",
+			name: "vhdx",
 			imf:  Vhdx,
-			want: "vhdx.zip",
+			want: "vhdx.zst",
 		},
 		{
 			name: "qcow2",
 			imf:  Qcow,
-			want: "qcow2.xz",
+			want: "qcow2.zst",
 		},
 		{
-			name: "raw.gz",
+			name: "raw",
 			imf:  Raw,
-			want: "raw.gz",
+			want: "raw.zst",
 		}, {
 			name: "tar.xz",
 			imf:  Tar,

pkg/machine/ignition/ignition.go

@@ -178,24 +178,6 @@ ExecStart=
 ExecStart=-/usr/sbin/agetty --autologin root --noclear %I $TERM
 `
 
-	deMoby := parser.NewUnitFile()
-	deMoby.Add("Unit", "Description", "Remove moby-engine")
-	deMoby.Add("Unit", "After", "systemd-machine-id-commit.service")
-	deMoby.Add("Unit", "Before", "zincati.service")
-	deMoby.Add("Unit", "ConditionPathExists", "!/var/lib/%N.stamp")
-
-	deMoby.Add("Service", "Type", "oneshot")
-	deMoby.Add("Service", "RemainAfterExit", "yes")
-	deMoby.Add("Service", "ExecStart", "/usr/bin/rpm-ostree override remove moby-engine")
-	deMoby.Add("Service", "ExecStart", "/usr/bin/rpm-ostree ex apply-live --allow-replacement")
-	deMoby.Add("Service", "ExecStartPost", "/bin/touch /var/lib/%N.stamp")
-
-	deMoby.Add("Install", "WantedBy", "default.target")
-	deMobyFile, err := deMoby.ToString()
-	if err != nil {
-		return err
-	}
-
 	// This service gets environment variables that are provided
 	// through qemu fw_cfg and then sets them into systemd/system.conf.d,
 	// profile.d and environment.d files
@@ -252,11 +234,6 @@ ExecStart=-/usr/sbin/agetty --autologin root --noclear %I $TERM
 				Name:    "docker.socket",
 				Mask:    BoolToPtr(true),
 			},
-			{
-				Enabled:  BoolToPtr(true),
-				Name:     "remove-moby.service",
-				Contents: &deMobyFile,
-			},
 			{
 				// Disable auto-updating of fcos images
 				// https://github.com/containers/podman/issues/20122
@@ -871,7 +848,7 @@ func GetNetRecoveryUnitFile() *parser.UnitFile {
 
 func DefaultReadyUnitFile() parser.UnitFile {
 	u := parser.NewUnitFile()
-	u.Add("Unit", "After", "remove-moby.service sshd.socket sshd.service")
+	u.Add("Unit", "After", "sshd.socket sshd.service")
 	u.Add("Unit", "OnFailure", "emergency.target")
 	u.Add("Unit", "OnFailureJobMode", "isolate")
 	u.Add("Service", "Type", "oneshot")

pkg/machine/ocipull/ociartifact.go

@@ -27,7 +27,7 @@ const (
 	// TODO This is temporary until we decide on a proper image name
 	artifactRegistry     = "quay.io"
 	artifactRepo         = "baude"
-	artifactImageName    = "podman-machine-images-art"
+	artifactImageName    = "stage-podman-machine"
 	artifactOriginalName = "org.opencontainers.image.title"
 	machineOS            = "linux"
 )