mirror of
https://github.com/containers/podman.git
synced 2025-10-19 20:23:08 +08:00
zstd now default compression for podman machine
given that we are moving to building our own machine images, we have decided to use zstd compression as it is superior in speed to the alternatives. as such, this pr adds zstd to our machine code; and also has to account for dealing with sparseness on darwin; which the default zstd golang library does not. [NO NEW TESTS NEEDED] Signed-off-by: Brent Baude <bbaude@redhat.com>
This commit is contained in:
2
go.mod
2
go.mod
@ -41,6 +41,7 @@ require (
|
|||||||
github.com/hashicorp/go-multierror v1.1.1
|
github.com/hashicorp/go-multierror v1.1.1
|
||||||
github.com/hugelgupf/p9 v0.3.1-0.20230822151754-54f5c5530921
|
github.com/hugelgupf/p9 v0.3.1-0.20230822151754-54f5c5530921
|
||||||
github.com/json-iterator/go v1.1.12
|
github.com/json-iterator/go v1.1.12
|
||||||
|
github.com/klauspost/compress v1.17.5
|
||||||
github.com/linuxkit/virtsock v0.0.0-20220523201153-1a23e78aa7a2
|
github.com/linuxkit/virtsock v0.0.0-20220523201153-1a23e78aa7a2
|
||||||
github.com/mattn/go-shellwords v1.0.12
|
github.com/mattn/go-shellwords v1.0.12
|
||||||
github.com/mattn/go-sqlite3 v1.14.21
|
github.com/mattn/go-sqlite3 v1.14.21
|
||||||
@ -149,7 +150,6 @@ require (
|
|||||||
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
||||||
github.com/jinzhu/copier v0.4.0 // indirect
|
github.com/jinzhu/copier v0.4.0 // indirect
|
||||||
github.com/josharian/intern v1.0.0 // indirect
|
github.com/josharian/intern v1.0.0 // indirect
|
||||||
github.com/klauspost/compress v1.17.5 // indirect
|
|
||||||
github.com/klauspost/cpuid/v2 v2.2.6 // indirect
|
github.com/klauspost/cpuid/v2 v2.2.6 // indirect
|
||||||
github.com/klauspost/pgzip v1.2.6 // indirect
|
github.com/klauspost/pgzip v1.2.6 // indirect
|
||||||
github.com/kr/fs v0.1.0 // indirect
|
github.com/kr/fs v0.1.0 // indirect
|
||||||
|
@ -33,11 +33,11 @@ func Test_compressionFromFile(t *testing.T) {
|
|||||||
want: Bz2,
|
want: Bz2,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "default is xz",
|
name: "default is zstd",
|
||||||
args: args{
|
args: args{
|
||||||
path: "/tmp/foo",
|
path: "/tmp/foo",
|
||||||
},
|
},
|
||||||
want: Xz,
|
want: Zstd,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
for _, tt := range tests {
|
for _, tt := range tests {
|
||||||
@ -76,9 +76,9 @@ func TestImageCompression_String(t *testing.T) {
|
|||||||
want: "zip",
|
want: "zip",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "xz is default",
|
name: "zstd is default",
|
||||||
c: 99,
|
c: 99,
|
||||||
want: "xz",
|
want: "zst",
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
for _, tt := range tests {
|
for _, tt := range tests {
|
||||||
|
@ -9,6 +9,7 @@ const (
|
|||||||
Zip
|
Zip
|
||||||
Gz
|
Gz
|
||||||
Bz2
|
Bz2
|
||||||
|
Zstd
|
||||||
)
|
)
|
||||||
|
|
||||||
func KindFromFile(path string) ImageCompression {
|
func KindFromFile(path string) ImageCompression {
|
||||||
@ -19,8 +20,10 @@ func KindFromFile(path string) ImageCompression {
|
|||||||
return Gz
|
return Gz
|
||||||
case strings.HasSuffix(path, Zip.String()):
|
case strings.HasSuffix(path, Zip.String()):
|
||||||
return Zip
|
return Zip
|
||||||
}
|
case strings.HasSuffix(path, Xz.String()):
|
||||||
return Xz
|
return Xz
|
||||||
|
}
|
||||||
|
return Zstd
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c ImageCompression) String() string {
|
func (c ImageCompression) String() string {
|
||||||
@ -31,6 +34,8 @@ func (c ImageCompression) String() string {
|
|||||||
return "zip"
|
return "zip"
|
||||||
case Bz2:
|
case Bz2:
|
||||||
return "bz2"
|
return "bz2"
|
||||||
}
|
case Xz:
|
||||||
return "xz"
|
return "xz"
|
||||||
|
}
|
||||||
|
return "zst"
|
||||||
}
|
}
|
||||||
|
@ -18,6 +18,7 @@ import (
|
|||||||
"github.com/containers/podman/v5/utils"
|
"github.com/containers/podman/v5/utils"
|
||||||
"github.com/containers/storage/pkg/archive"
|
"github.com/containers/storage/pkg/archive"
|
||||||
crcOs "github.com/crc-org/crc/v2/pkg/os"
|
crcOs "github.com/crc-org/crc/v2/pkg/os"
|
||||||
|
"github.com/klauspost/compress/zstd"
|
||||||
"github.com/sirupsen/logrus"
|
"github.com/sirupsen/logrus"
|
||||||
"github.com/ulikunitz/xz"
|
"github.com/ulikunitz/xz"
|
||||||
)
|
)
|
||||||
@ -59,12 +60,22 @@ func Decompress(localPath *define.VMFile, uncompressedPath string) error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
// darwin really struggles with sparse files. being diligent here
|
||||||
fmt.Printf("Copying uncompressed file %q to %q/n", localPath.GetPath(), dstFile.Name())
|
fmt.Printf("Copying uncompressed file %q to %q/n", localPath.GetPath(), dstFile.Name())
|
||||||
|
|
||||||
|
// Keeping CRC implementation for now, but ideally this could be pruned and
|
||||||
|
// sparsewriter could be used. in that case, this area needs rework or
|
||||||
|
// sparsewriter be made to honor the *file interface
|
||||||
_, err = crcOs.CopySparse(uncompressedFileWriter, dstFile)
|
_, err = crcOs.CopySparse(uncompressedFileWriter, dstFile)
|
||||||
return err
|
return err
|
||||||
case archive.Gzip:
|
case archive.Gzip:
|
||||||
if runtime.GOOS == "darwin" {
|
if runtime.GOOS == "darwin" {
|
||||||
return decompressGzWithSparse(prefix, localPath, uncompressedPath)
|
return decompressGzWithSparse(prefix, localPath, uncompressedFileWriter)
|
||||||
|
}
|
||||||
|
fallthrough
|
||||||
|
case archive.Zstd:
|
||||||
|
if runtime.GOOS == "darwin" {
|
||||||
|
return decompressZstdWithSparse(prefix, localPath, uncompressedFileWriter)
|
||||||
}
|
}
|
||||||
fallthrough
|
fallthrough
|
||||||
default:
|
default:
|
||||||
@ -225,22 +236,31 @@ func decompressZip(prefix string, src string, output io.WriteCloser) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
func decompressGzWithSparse(prefix string, compressedPath *define.VMFile, uncompressedPath string) error {
|
func decompressWithSparse(prefix string, compressedReader io.Reader, uncompressedFile *os.File) error {
|
||||||
stat, err := os.Stat(compressedPath.GetPath())
|
dstFile := NewSparseWriter(uncompressedFile)
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
dstFile, err := os.OpenFile(uncompressedPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, stat.Mode())
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
defer func() {
|
defer func() {
|
||||||
if err := dstFile.Close(); err != nil {
|
if err := dstFile.Close(); err != nil {
|
||||||
logrus.Errorf("unable to close uncompressed file %s: %q", uncompressedPath, err)
|
logrus.Errorf("unable to close uncompressed file %s: %q", uncompressedFile.Name(), err)
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
|
// TODO remove the following line when progress bars work
|
||||||
|
_ = prefix
|
||||||
|
// p, bar := utils.ProgressBar(prefix, stat.Size(), prefix+": done")
|
||||||
|
// proxyReader := bar.ProxyReader(f)
|
||||||
|
// defer func() {
|
||||||
|
// if err := proxyReader.Close(); err != nil {
|
||||||
|
// logrus.Error(err)
|
||||||
|
// }
|
||||||
|
// }()
|
||||||
|
|
||||||
|
// p.Wait()
|
||||||
|
_, err := io.Copy(dstFile, compressedReader)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func decompressGzWithSparse(prefix string, compressedPath *define.VMFile, uncompressedFileWriter *os.File) error {
|
||||||
|
logrus.Debugf("decompressing %s", compressedPath.GetPath())
|
||||||
f, err := os.Open(compressedPath.GetPath())
|
f, err := os.Open(compressedPath.GetPath())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
@ -260,20 +280,34 @@ func decompressGzWithSparse(prefix string, compressedPath *define.VMFile, uncomp
|
|||||||
logrus.Errorf("unable to close gzreader: %q", err)
|
logrus.Errorf("unable to close gzreader: %q", err)
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
// This way we get something to look at in debug mode
|
||||||
// TODO remove the following line when progress bars work
|
defer func() {
|
||||||
_ = prefix
|
|
||||||
// p, bar := utils.ProgressBar(prefix, stat.Size(), prefix+": done")
|
|
||||||
// proxyReader := bar.ProxyReader(f)
|
|
||||||
// defer func() {
|
|
||||||
// if err := proxyReader.Close(); err != nil {
|
|
||||||
// logrus.Error(err)
|
|
||||||
// }
|
|
||||||
// }()
|
|
||||||
|
|
||||||
logrus.Debugf("decompressing %s", compressedPath.GetPath())
|
|
||||||
_, err = crcOs.CopySparse(dstFile, gzReader)
|
|
||||||
logrus.Debug("decompression complete")
|
logrus.Debug("decompression complete")
|
||||||
// p.Wait()
|
}()
|
||||||
return err
|
return decompressWithSparse(prefix, gzReader, uncompressedFileWriter)
|
||||||
|
}
|
||||||
|
|
||||||
|
func decompressZstdWithSparse(prefix string, compressedPath *define.VMFile, uncompressedFileWriter *os.File) error {
|
||||||
|
logrus.Debugf("decompressing %s", compressedPath.GetPath())
|
||||||
|
f, err := os.Open(compressedPath.GetPath())
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer func() {
|
||||||
|
if err := f.Close(); err != nil {
|
||||||
|
logrus.Errorf("unable to close on compressed file %s: %q", compressedPath.GetPath(), err)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
zstdReader, err := zstd.NewReader(f)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer zstdReader.Close()
|
||||||
|
|
||||||
|
// This way we get something to look at in debug mode
|
||||||
|
defer func() {
|
||||||
|
logrus.Debug("decompression complete")
|
||||||
|
}()
|
||||||
|
return decompressWithSparse(prefix, zstdReader, uncompressedFileWriter)
|
||||||
}
|
}
|
||||||
|
@ -1,5 +1,7 @@
|
|||||||
package define
|
package define
|
||||||
|
|
||||||
|
import "fmt"
|
||||||
|
|
||||||
type ImageFormat int64
|
type ImageFormat int64
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@ -22,13 +24,9 @@ func (imf ImageFormat) Kind() string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (imf ImageFormat) KindWithCompression() string {
|
func (imf ImageFormat) KindWithCompression() string {
|
||||||
switch imf {
|
// Tar uses xz; all others use zstd
|
||||||
case Vhdx:
|
if imf == Tar {
|
||||||
return "vhdx.zip"
|
|
||||||
case Tar:
|
|
||||||
return "tar.xz"
|
return "tar.xz"
|
||||||
case Raw:
|
|
||||||
return "raw.gz"
|
|
||||||
}
|
}
|
||||||
return "qcow2.xz"
|
return fmt.Sprintf("%s.zst", imf.Kind())
|
||||||
}
|
}
|
||||||
|
@ -45,19 +45,19 @@ func TestImageFormat_KindWithCompression(t *testing.T) {
|
|||||||
want string
|
want string
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "vhdx.zip",
|
name: "vhdx",
|
||||||
imf: Vhdx,
|
imf: Vhdx,
|
||||||
want: "vhdx.zip",
|
want: "vhdx.zst",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "qcow2",
|
name: "qcow2",
|
||||||
imf: Qcow,
|
imf: Qcow,
|
||||||
want: "qcow2.xz",
|
want: "qcow2.zst",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "raw.gz",
|
name: "raw",
|
||||||
imf: Raw,
|
imf: Raw,
|
||||||
want: "raw.gz",
|
want: "raw.zst",
|
||||||
}, {
|
}, {
|
||||||
name: "tar.xz",
|
name: "tar.xz",
|
||||||
imf: Tar,
|
imf: Tar,
|
||||||
|
@ -178,24 +178,6 @@ ExecStart=
|
|||||||
ExecStart=-/usr/sbin/agetty --autologin root --noclear %I $TERM
|
ExecStart=-/usr/sbin/agetty --autologin root --noclear %I $TERM
|
||||||
`
|
`
|
||||||
|
|
||||||
deMoby := parser.NewUnitFile()
|
|
||||||
deMoby.Add("Unit", "Description", "Remove moby-engine")
|
|
||||||
deMoby.Add("Unit", "After", "systemd-machine-id-commit.service")
|
|
||||||
deMoby.Add("Unit", "Before", "zincati.service")
|
|
||||||
deMoby.Add("Unit", "ConditionPathExists", "!/var/lib/%N.stamp")
|
|
||||||
|
|
||||||
deMoby.Add("Service", "Type", "oneshot")
|
|
||||||
deMoby.Add("Service", "RemainAfterExit", "yes")
|
|
||||||
deMoby.Add("Service", "ExecStart", "/usr/bin/rpm-ostree override remove moby-engine")
|
|
||||||
deMoby.Add("Service", "ExecStart", "/usr/bin/rpm-ostree ex apply-live --allow-replacement")
|
|
||||||
deMoby.Add("Service", "ExecStartPost", "/bin/touch /var/lib/%N.stamp")
|
|
||||||
|
|
||||||
deMoby.Add("Install", "WantedBy", "default.target")
|
|
||||||
deMobyFile, err := deMoby.ToString()
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
// This service gets environment variables that are provided
|
// This service gets environment variables that are provided
|
||||||
// through qemu fw_cfg and then sets them into systemd/system.conf.d,
|
// through qemu fw_cfg and then sets them into systemd/system.conf.d,
|
||||||
// profile.d and environment.d files
|
// profile.d and environment.d files
|
||||||
@ -252,11 +234,6 @@ ExecStart=-/usr/sbin/agetty --autologin root --noclear %I $TERM
|
|||||||
Name: "docker.socket",
|
Name: "docker.socket",
|
||||||
Mask: BoolToPtr(true),
|
Mask: BoolToPtr(true),
|
||||||
},
|
},
|
||||||
{
|
|
||||||
Enabled: BoolToPtr(true),
|
|
||||||
Name: "remove-moby.service",
|
|
||||||
Contents: &deMobyFile,
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
// Disable auto-updating of fcos images
|
// Disable auto-updating of fcos images
|
||||||
// https://github.com/containers/podman/issues/20122
|
// https://github.com/containers/podman/issues/20122
|
||||||
@ -871,7 +848,7 @@ func GetNetRecoveryUnitFile() *parser.UnitFile {
|
|||||||
|
|
||||||
func DefaultReadyUnitFile() parser.UnitFile {
|
func DefaultReadyUnitFile() parser.UnitFile {
|
||||||
u := parser.NewUnitFile()
|
u := parser.NewUnitFile()
|
||||||
u.Add("Unit", "After", "remove-moby.service sshd.socket sshd.service")
|
u.Add("Unit", "After", "sshd.socket sshd.service")
|
||||||
u.Add("Unit", "OnFailure", "emergency.target")
|
u.Add("Unit", "OnFailure", "emergency.target")
|
||||||
u.Add("Unit", "OnFailureJobMode", "isolate")
|
u.Add("Unit", "OnFailureJobMode", "isolate")
|
||||||
u.Add("Service", "Type", "oneshot")
|
u.Add("Service", "Type", "oneshot")
|
||||||
|
@ -27,7 +27,7 @@ const (
|
|||||||
// TODO This is temporary until we decide on a proper image name
|
// TODO This is temporary until we decide on a proper image name
|
||||||
artifactRegistry = "quay.io"
|
artifactRegistry = "quay.io"
|
||||||
artifactRepo = "baude"
|
artifactRepo = "baude"
|
||||||
artifactImageName = "podman-machine-images-art"
|
artifactImageName = "stage-podman-machine"
|
||||||
artifactOriginalName = "org.opencontainers.image.title"
|
artifactOriginalName = "org.opencontainers.image.title"
|
||||||
machineOS = "linux"
|
machineOS = "linux"
|
||||||
)
|
)
|
||||||
|
Reference in New Issue
Block a user