opensource/podman
1
0
Fork 0
mirror of https://github.com/containers/podman.git synced 2025-10-17 19:24:04 +08:00
Code Issues Packages Projects Releases Wiki Activity

zstd now default compression for podman machine

Browse Source 
given that we are moving to building our own machine images, we have
decided to use zstd compression as it is superior in speed to the
alternatives.  as such, this pr adds zstd to our machine code; and also
has to account for dealing with sparseness on darwin; which the default
zstd golang library does not.

[NO NEW TESTS NEEDED]

Signed-off-by: Brent Baude <bbaude@redhat.com>
This commit is contained in:
Brent Baude
2024-02-16 09:47:12 -06:00
parent 1d4651bccc
commit a31e8d2a23
8 changed files with 86 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
pkg/machine/compression/compression_test.go
View File

@ -33,11 +33,11 @@ func Test_compressionFromFile(t *testing.T) {
want: Bz2,
},
{
name: "default is xz",
name: "default is zstd",
args: args{
path: "/tmp/foo",
},
want: Xz,
want: Zstd,
},
}
for _, tt := range tests {
@ -76,9 +76,9 @@ func TestImageCompression_String(t *testing.T) {
want: "zip",
},
{
name: "xz is default",
name: "zstd is default",
c: 99,
want: "xz",
want: "zst",
},
}
for _, tt := range tests {

9
pkg/machine/compression/config.go
View File

@ -9,6 +9,7 @@ const (
Zip
Gz
Bz2
Zstd
)
func KindFromFile(path string) ImageCompression {
@ -19,8 +20,10 @@ func KindFromFile(path string) ImageCompression {
return Gz
case strings.HasSuffix(path, Zip.String()):
return Zip
case strings.HasSuffix(path, Xz.String()):
return Xz
}
return Xz
return Zstd
}
func (c ImageCompression) String() string {
@ -31,6 +34,8 @@ func (c ImageCompression) String() string {
return "zip"
case Bz2:
return "bz2"
case Xz:
return "xz"
}
return "xz"
return "zst"
}

90
pkg/machine/compression/decompress.go
View File

@ -18,6 +18,7 @@ import (
"github.com/containers/podman/v5/utils"
"github.com/containers/storage/pkg/archive"
crcOs "github.com/crc-org/crc/v2/pkg/os"
"github.com/klauspost/compress/zstd"
"github.com/sirupsen/logrus"
"github.com/ulikunitz/xz"
)
@ -59,12 +60,22 @@ func Decompress(localPath *define.VMFile, uncompressedPath string) error {
if err != nil {
return err
}
// darwin really struggles with sparse files. being diligent here
fmt.Printf("Copying uncompressed file %q to %q/n", localPath.GetPath(), dstFile.Name())
// Keeping CRC implementation for now, but ideally this could be pruned and
// sparsewriter could be used. in that case, this area needs rework or
// sparsewriter be made to honor the *file interface
_, err = crcOs.CopySparse(uncompressedFileWriter, dstFile)
return err
case archive.Gzip:
if runtime.GOOS == "darwin" {
return decompressGzWithSparse(prefix, localPath, uncompressedPath)
return decompressGzWithSparse(prefix, localPath, uncompressedFileWriter)
}
fallthrough
case archive.Zstd:
if runtime.GOOS == "darwin" {
return decompressZstdWithSparse(prefix, localPath, uncompressedFileWriter)
}
fallthrough
default:
@ -225,22 +236,31 @@ func decompressZip(prefix string, src string, output io.WriteCloser) error {
return err
}
func decompressGzWithSparse(prefix string, compressedPath *define.VMFile, uncompressedPath string) error {
stat, err := os.Stat(compressedPath.GetPath())
if err != nil {
return err
}
dstFile, err := os.OpenFile(uncompressedPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, stat.Mode())
if err != nil {
return err
}
func decompressWithSparse(prefix string, compressedReader io.Reader, uncompressedFile *os.File) error {
dstFile := NewSparseWriter(uncompressedFile)
defer func() {
if err := dstFile.Close(); err != nil {
logrus.Errorf("unable to close uncompressed file %s: %q", uncompressedPath, err)
logrus.Errorf("unable to close uncompressed file %s: %q", uncompressedFile.Name(), err)
}
}()
// TODO remove the following line when progress bars work
_ = prefix
// p, bar := utils.ProgressBar(prefix, stat.Size(), prefix+": done")
// proxyReader := bar.ProxyReader(f)
// defer func() {
// if err := proxyReader.Close(); err != nil {
// logrus.Error(err)
// }
// }()
// p.Wait()
_, err := io.Copy(dstFile, compressedReader)
return err
}
func decompressGzWithSparse(prefix string, compressedPath *define.VMFile, uncompressedFileWriter *os.File) error {
logrus.Debugf("decompressing %s", compressedPath.GetPath())
f, err := os.Open(compressedPath.GetPath())
if err != nil {
return err
@ -260,20 +280,34 @@ func decompressGzWithSparse(prefix string, compressedPath *define.VMFile, uncomp
logrus.Errorf("unable to close gzreader: %q", err)
}
}()
// TODO remove the following line when progress bars work
_ = prefix
// p, bar := utils.ProgressBar(prefix, stat.Size(), prefix+": done")
// proxyReader := bar.ProxyReader(f)
// defer func() {
// if err := proxyReader.Close(); err != nil {
// logrus.Error(err)
// }
// }()
logrus.Debugf("decompressing %s", compressedPath.GetPath())
_, err = crcOs.CopySparse(dstFile, gzReader)
logrus.Debug("decompression complete")
// p.Wait()
return err
// This way we get something to look at in debug mode
defer func() {
logrus.Debug("decompression complete")
}()
return decompressWithSparse(prefix, gzReader, uncompressedFileWriter)
}
func decompressZstdWithSparse(prefix string, compressedPath *define.VMFile, uncompressedFileWriter *os.File) error {
logrus.Debugf("decompressing %s", compressedPath.GetPath())
f, err := os.Open(compressedPath.GetPath())
if err != nil {
return err
}
defer func() {
if err := f.Close(); err != nil {
logrus.Errorf("unable to close on compressed file %s: %q", compressedPath.GetPath(), err)
}
}()
zstdReader, err := zstd.NewReader(f)
if err != nil {
return err
}
defer zstdReader.Close()
// This way we get something to look at in debug mode
defer func() {
logrus.Debug("decompression complete")
}()
return decompressWithSparse(prefix, zstdReader, uncompressedFileWriter)
}