Merge pull request #25305 from Luap99/artifact-reflink

artifact extract: support reflink copy

go.mod

@@ -21,7 +21,7 @@ require (
 	github.com/containers/libhvee v0.9.0
 	github.com/containers/ocicrypt v1.2.1
 	github.com/containers/psgo v1.9.0
-	github.com/containers/storage v1.57.1
+	github.com/containers/storage v1.57.2-0.20250211190637-7aa96daee0a3
 	github.com/containers/winquit v1.1.0
 	github.com/coreos/go-systemd/v22 v22.5.1-0.20231103132048-7d375ecc2b09
 	github.com/crc-org/crc/v2 v2.45.0
@@ -211,7 +211,7 @@ require (
 	github.com/u-root/uio v0.0.0-20240224005618-d2acac8f3701 // indirect
 	github.com/ugorji/go/codec v1.2.12 // indirect
 	github.com/ulikunitz/xz v0.5.12 // indirect
-	github.com/vbatts/tar-split v0.11.7 // indirect
+	github.com/vbatts/tar-split v0.12.1 // indirect
 	github.com/vishvananda/netns v0.0.4 // indirect
 	github.com/yusufpapurcu/wmi v1.2.4 // indirect
 	go.mongodb.org/mongo-driver v1.14.0 // indirect

go.sum

@@ -96,8 +96,8 @@ github.com/containers/ocicrypt v1.2.1 h1:0qIOTT9DoYwcKmxSt8QJt+VzMY18onl9jUXsxpV
 github.com/containers/ocicrypt v1.2.1/go.mod h1:aD0AAqfMp0MtwqWgHM1bUwe1anx0VazI108CRrSKINQ=
 github.com/containers/psgo v1.9.0 h1:eJ74jzSaCHnWt26OlKZROSyUyRcGDf+gYBdXnxrMW4g=
 github.com/containers/psgo v1.9.0/go.mod h1:0YoluUm43Mz2UnBIh1P+6V6NWcbpTL5uRtXyOcH0B5A=
-github.com/containers/storage v1.57.1 h1:hKPoFsuBcB3qTzBxa4IFpZMRzUuL5Xhv/BE44W0XHx8=
-github.com/containers/storage v1.57.1/go.mod h1:i/Hb4lu7YgFr9G0K6BMjqW0BLJO1sFsnWQwj2UoWCUM=
+github.com/containers/storage v1.57.2-0.20250211190637-7aa96daee0a3 h1:YLjd5aplmRP98Jlrqz5+kNmbVZvpZwrZygkF96KR2Fs=
+github.com/containers/storage v1.57.2-0.20250211190637-7aa96daee0a3/go.mod h1:zsh6czcxcdqKIz//cVU6waEJ+2Ui8OEnrwCvM/DE3iU=
 github.com/containers/winquit v1.1.0 h1:jArun04BNDQvt2W0Y78kh9TazN2EIEMG5Im6/JY7+pE=
 github.com/containers/winquit v1.1.0/go.mod h1:PsPeZlnbkmGGIToMPHF1zhWjBUkd8aHjMOr/vFcPxw8=
 github.com/coreos/go-oidc/v3 v3.12.0 h1:sJk+8G2qq94rDI6ehZ71Bol3oUHy63qNYmkiSjrc/Jo=
@@ -514,8 +514,8 @@ github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65E
 github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
 github.com/ulikunitz/xz v0.5.12 h1:37Nm15o69RwBkXM0J6A5OlE67RZTfzUxTj8fB3dfcsc=
 github.com/ulikunitz/xz v0.5.12/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
-github.com/vbatts/tar-split v0.11.7 h1:ixZ93pO/GmvaZw4Vq9OwmfZK/kc2zKdPfu0B+gYqs3U=
-github.com/vbatts/tar-split v0.11.7/go.mod h1:eF6B6i6ftWQcDqEn3/iGFRFRo8cBIMSJVOpnNdfTMFA=
+github.com/vbatts/tar-split v0.12.1 h1:CqKoORW7BUWBe7UL/iqTVvkTBOF8UvOMKOIZykxnnbo=
+github.com/vbatts/tar-split v0.12.1/go.mod h1:eF6B6i6ftWQcDqEn3/iGFRFRo8cBIMSJVOpnNdfTMFA=
 github.com/vbauerster/mpb/v8 v8.9.2 h1:kb91+D643Qg040bbICYtzpjgZ9ypVO/+sjv4Jcm6si4=
 github.com/vbauerster/mpb/v8 v8.9.2/go.mod h1:hxS8Hz4C6ijnppDSIX6LjG8FYJSoPo9iIOcE53Zik0c=
 github.com/vishvananda/netlink v1.3.1-0.20250128002108-7c2350bd140f h1:G5t3qYQ3YL2zMn2kFzRYIPk1EvDvMNV9pP+w+39VtzI=

pkg/libartifact/store/store.go

@@ -466,7 +466,12 @@ func copyImageBlobToFile(ctx context.Context, imgSrc types.ImageSource, digest d
 	}
 	defer dest.Close()
 
-	// TODO use reflink is possible
+	// By default the c/image oci layout API for GetBlob() should always return a os.File in our usage here.
+	// And since it is a file we can try to reflink it. In case it is not we should default to the normal copy.
+	if file, ok := src.(*os.File); ok {
+		return fileutils.ReflinkOrCopy(file, dest)
+	}
+
 	_, err = io.Copy(dest, src)
 	return err
 }

vendor/github.com/containers/storage/.cirrus.yml

@@ -23,7 +23,7 @@ env:
     # GCE project where images live
     IMAGE_PROJECT: "libpod-218412"
     # VM Image built in containers/automation_images
-    IMAGE_SUFFIX: "c20250107t132430z-f41f40d13"
+    IMAGE_SUFFIX: "c20250131t121915z-f41f40d13"
     FEDORA_CACHE_IMAGE_NAME: "fedora-${IMAGE_SUFFIX}"
     DEBIAN_CACHE_IMAGE_NAME: "debian-${IMAGE_SUFFIX}"
 

vendor/github.com/containers/storage/VERSION

@@ -1 +1 @@
-1.57.1
+1.58.0-dev

vendor/github.com/containers/storage/drivers/overlay/overlay.go

@@ -36,7 +36,6 @@ import (
 	"github.com/containers/storage/pkg/system"
 	"github.com/containers/storage/pkg/unshare"
 	units "github.com/docker/go-units"
-	"github.com/hashicorp/go-multierror"
 	digest "github.com/opencontainers/go-digest"
 	"github.com/opencontainers/selinux/go-selinux"
 	"github.com/opencontainers/selinux/go-selinux/label"
@@ -419,7 +418,7 @@ func Init(home string, options graphdriver.Options) (graphdriver.Driver, error)
 
 	if !opts.skipMountHome {
 		if err := mount.MakePrivate(home); err != nil {
-			return nil, err
+			return nil, fmt.Errorf("overlay: failed to make mount private: %w", err)
 		}
 	}
 
@@ -1343,7 +1342,7 @@ func (d *Driver) recreateSymlinks() error {
 		return err
 	}
 	// Keep looping as long as we take some corrective action in each iteration
-	var errs *multierror.Error
+	var errs error
 	madeProgress := true
 	iterations := 0
 	for madeProgress {
@@ -1359,7 +1358,7 @@ func (d *Driver) recreateSymlinks() error {
 			// Read the "link" file under each layer to get the name of the symlink
 			data, err := os.ReadFile(path.Join(d.dir(dir.Name()), "link"))
 			if err != nil {
-				errs = multierror.Append(errs, fmt.Errorf("reading name of symlink for %q: %w", dir.Name(), err))
+				errs = errors.Join(errs, fmt.Errorf("reading name of symlink for %q: %w", dir.Name(), err))
 				continue
 			}
 			linkPath := path.Join(d.home, linkDir, strings.Trim(string(data), "\n"))
@@ -1368,12 +1367,12 @@ func (d *Driver) recreateSymlinks() error {
 			err = fileutils.Lexists(linkPath)
 			if err != nil && os.IsNotExist(err) {
 				if err := os.Symlink(path.Join("..", dir.Name(), "diff"), linkPath); err != nil {
-					errs = multierror.Append(errs, err)
+					errs = errors.Join(errs, err)
 					continue
 				}
 				madeProgress = true
 			} else if err != nil {
-				errs = multierror.Append(errs, err)
+				errs = errors.Join(errs, err)
 				continue
 			}
 		}
@@ -1384,7 +1383,7 @@ func (d *Driver) recreateSymlinks() error {
 		// that each symlink we have corresponds to one.
 		links, err := os.ReadDir(linkDirFullPath)
 		if err != nil {
-			errs = multierror.Append(errs, err)
+			errs = errors.Join(errs, err)
 			continue
 		}
 		// Go through all of the symlinks in the "l" directory
@@ -1392,16 +1391,16 @@ func (d *Driver) recreateSymlinks() error {
 			// Read the symlink's target, which should be "../$layer/diff"
 			target, err := os.Readlink(filepath.Join(linkDirFullPath, link.Name()))
 			if err != nil {
-				errs = multierror.Append(errs, err)
+				errs = errors.Join(errs, err)
 				continue
 			}
 			targetComponents := strings.Split(target, string(os.PathSeparator))
 			if len(targetComponents) != 3 || targetComponents[0] != ".." || targetComponents[2] != "diff" {
-				errs = multierror.Append(errs, fmt.Errorf("link target of %q looks weird: %q", link, target))
+				errs = errors.Join(errs, fmt.Errorf("link target of %q looks weird: %q", link, target))
 				// force the link to be recreated on the next pass
 				if err := os.Remove(filepath.Join(linkDirFullPath, link.Name())); err != nil {
 					if !os.IsNotExist(err) {
-						errs = multierror.Append(errs, fmt.Errorf("removing link %q: %w", link, err))
+						errs = errors.Join(errs, fmt.Errorf("removing link %q: %w", link, err))
 					} // else don’t report any error, but also don’t set madeProgress.
 					continue
 				}
@@ -1417,7 +1416,7 @@ func (d *Driver) recreateSymlinks() error {
 				// NOTE: If two or more links point to the same target, we will update linkFile
 				// with every value of link.Name(), and set madeProgress = true every time.
 				if err := os.WriteFile(linkFile, []byte(link.Name()), 0o644); err != nil {
-					errs = multierror.Append(errs, fmt.Errorf("correcting link for layer %s: %w", targetID, err))
+					errs = errors.Join(errs, fmt.Errorf("correcting link for layer %s: %w", targetID, err))
 					continue
 				}
 				madeProgress = true
@@ -1425,14 +1424,11 @@ func (d *Driver) recreateSymlinks() error {
 		}
 		iterations++
 		if iterations >= maxIterations {
-			errs = multierror.Append(errs, fmt.Errorf("reached %d iterations in overlay graph driver’s recreateSymlink, giving up", iterations))
+			errs = errors.Join(errs, fmt.Errorf("reached %d iterations in overlay graph driver’s recreateSymlink, giving up", iterations))
 			break
 		}
 	}
-	if errs != nil {
-		return errs.ErrorOrNil()
-	}
-	return nil
+	return errs
 }
 
 // Get creates and mounts the required file system for the given id and returns the mount path.
@@ -2103,17 +2099,16 @@ func (g *overlayFileGetter) Get(path string) (io.ReadCloser, error) {
 	return nil, fmt.Errorf("%s: %w", path, os.ErrNotExist)
 }
 
-func (g *overlayFileGetter) Close() error {
-	var errs *multierror.Error
+func (g *overlayFileGetter) Close() (errs error) {
 	for _, f := range g.composefsMounts {
 		if err := f.Close(); err != nil {
-			errs = multierror.Append(errs, err)
+			errs = errors.Join(errs, err)
 		}
 		if err := unix.Rmdir(f.Name()); err != nil {
-			errs = multierror.Append(errs, err)
+			errs = errors.Join(errs, err)
 		}
 	}
-	return errs.ErrorOrNil()
+	return errs
 }
 
 // newStagingDir creates a new staging directory and returns the path to it.

vendor/github.com/containers/storage/layers.go

@@ -26,7 +26,6 @@ import (
 	"github.com/containers/storage/pkg/system"
 	"github.com/containers/storage/pkg/tarlog"
 	"github.com/containers/storage/pkg/truncindex"
-	multierror "github.com/hashicorp/go-multierror"
 	"github.com/klauspost/pgzip"
 	digest "github.com/opencontainers/go-digest"
 	"github.com/opencontainers/selinux/go-selinux"
@@ -937,7 +936,7 @@ func (r *layerStore) load(lockedForWriting bool) (bool, error) {
 				// Don't return the error immediately, because deleteInternal does not saveLayers();
 				// Even if deleting one incomplete layer fails, call saveLayers() so that other possible successfully
 				// deleted incomplete layers have their metadata correctly removed.
-				incompleteDeletionErrors = multierror.Append(incompleteDeletionErrors,
+				incompleteDeletionErrors = errors.Join(incompleteDeletionErrors,
 					fmt.Errorf("deleting layer %#v: %w", layer.ID, err))
 			}
 			modifiedLocations |= layerLocation(layer)
@@ -2256,33 +2255,33 @@ func (r *layerStore) Diff(from, to string, options *DiffOptions) (io.ReadCloser,
 	// but they modify in-memory state.
 	fgetter, err := r.newFileGetter(to)
 	if err != nil {
-		errs := multierror.Append(nil, fmt.Errorf("creating file-getter: %w", err))
+		errs := fmt.Errorf("creating file-getter: %w", err)
 		if err := decompressor.Close(); err != nil {
-			errs = multierror.Append(errs, fmt.Errorf("closing decompressor: %w", err))
+			errs = errors.Join(errs, fmt.Errorf("closing decompressor: %w", err))
 		}
 		if err := tsfile.Close(); err != nil {
-			errs = multierror.Append(errs, fmt.Errorf("closing tarstream headers: %w", err))
+			errs = errors.Join(errs, fmt.Errorf("closing tarstream headers: %w", err))
 		}
-		return nil, errs.ErrorOrNil()
+		return nil, errs
 	}
 
 	tarstream := asm.NewOutputTarStream(fgetter, metadata)
 	rc := ioutils.NewReadCloserWrapper(tarstream, func() error {
-		var errs *multierror.Error
+		var errs error
 		if err := decompressor.Close(); err != nil {
-			errs = multierror.Append(errs, fmt.Errorf("closing decompressor: %w", err))
+			errs = errors.Join(errs, fmt.Errorf("closing decompressor: %w", err))
 		}
 		if err := tsfile.Close(); err != nil {
-			errs = multierror.Append(errs, fmt.Errorf("closing tarstream headers: %w", err))
+			errs = errors.Join(errs, fmt.Errorf("closing tarstream headers: %w", err))
 		}
 		if err := tarstream.Close(); err != nil {
-			errs = multierror.Append(errs, fmt.Errorf("closing reconstructed tarstream: %w", err))
+			errs = errors.Join(errs, fmt.Errorf("closing reconstructed tarstream: %w", err))
 		}
 		if err := fgetter.Close(); err != nil {
-			errs = multierror.Append(errs, fmt.Errorf("closing file-getter: %w", err))
+			errs = errors.Join(errs, fmt.Errorf("closing file-getter: %w", err))
 		}
 		if errs != nil {
-			return errs.ErrorOrNil()
+			return errs
 		}
 		return nil
 	})

vendor/github.com/containers/storage/pkg/archive/archive.go

@@ -78,7 +78,6 @@ const (
 	windows = "windows"
 	darwin  = "darwin"
 	freebsd = "freebsd"
-	linux   = "linux"
 )
 
 var xattrsToIgnore = map[string]interface{}{

vendor/github.com/containers/storage/pkg/fileutils/reflink_linux.go

@@ -0,0 +1,20 @@
+package fileutils
+
+import (
+	"io"
+	"os"
+
+	"golang.org/x/sys/unix"
+)
+
+// ReflinkOrCopy attempts to reflink the source to the destination fd.
+// If reflinking fails or is unsupported, it falls back to io.Copy().
+func ReflinkOrCopy(src, dst *os.File) error {
+	err := unix.IoctlFileClone(int(dst.Fd()), int(src.Fd()))
+	if err == nil {
+		return nil
+	}
+
+	_, err = io.Copy(dst, src)
+	return err
+}

vendor/github.com/containers/storage/pkg/fileutils/reflink_unsupported.go

@@ -0,0 +1,15 @@
+//go:build !linux
+
+package fileutils
+
+import (
+	"io"
+	"os"
+)
+
+// ReflinkOrCopy attempts to reflink the source to the destination fd.
+// If reflinking fails or is unsupported, it falls back to io.Copy().
+func ReflinkOrCopy(src, dst *os.File) error {
+	_, err := io.Copy(dst, src)
+	return err
+}

vendor/github.com/containers/storage/pkg/unshare/unshare_linux.go

@@ -98,7 +98,7 @@ func IsSetID(path string, modeid os.FileMode, capid capability.Cap) (bool, error
 	return cap.Get(capability.EFFECTIVE, capid), nil
 }
 
-func (c *Cmd) Start() error {
+func (c *Cmd) Start() (retErr error) {
 	runtime.LockOSThread()
 	defer runtime.UnlockOSThread()
 
@@ -167,6 +167,15 @@ func (c *Cmd) Start() error {
 		return err
 	}
 
+	// If the function fails from here, we need to make sure the
+	// child process is killed and properly cleaned up.
+	defer func() {
+		if retErr != nil {
+			_ = c.Cmd.Process.Kill()
+			_ = c.Cmd.Wait()
+		}
+	}()
+
 	// Close the ends of the pipes that the parent doesn't need.
 	continueRead.Close()
 	continueRead = nil

vendor/github.com/containers/storage/store.go

@@ -18,6 +18,7 @@ import (
 
 	// register all of the built-in drivers
 	_ "github.com/containers/storage/drivers/register"
+	"golang.org/x/sync/errgroup"
 
 	drivers "github.com/containers/storage/drivers"
 	"github.com/containers/storage/internal/dedup"
@@ -30,7 +31,6 @@ import (
 	"github.com/containers/storage/pkg/stringutils"
 	"github.com/containers/storage/pkg/system"
 	"github.com/containers/storage/types"
-	"github.com/hashicorp/go-multierror"
 	digest "github.com/opencontainers/go-digest"
 	"github.com/opencontainers/selinux/go-selinux/label"
 	"github.com/sirupsen/logrus"
@@ -2744,7 +2744,7 @@ func (s *store) DeleteContainer(id string) error {
 			}
 		}
 
-		var wg multierror.Group
+		var wg errgroup.Group
 
 		middleDir := s.graphDriverName + "-containers"
 
@@ -2759,7 +2759,7 @@ func (s *store) DeleteContainer(id string) error {
 		})
 
 		if multierr := wg.Wait(); multierr != nil {
-			return multierr.ErrorOrNil()
+			return multierr
 		}
 		return s.containerStore.Delete(id)
 	})

vendor/github.com/vbatts/tar-split/archive/tar/writer.go

@@ -199,6 +199,9 @@ func (tw *Writer) writePAXHeader(hdr *Header, paxHdrs map[string]string) error {
 			flag = TypeXHeader
 		}
 		data := buf.String()
+		if len(data) > maxSpecialFileSize {
+			return ErrFieldTooLong
+		}
 		if err := tw.writeRawFile(name, data, flag, FormatPAX); err != nil || isGlobal {
 			return err // Global headers return here
 		}

vendor/modules.txt

@@ -364,7 +364,7 @@ github.com/containers/psgo/internal/dev
 github.com/containers/psgo/internal/host
 github.com/containers/psgo/internal/proc
 github.com/containers/psgo/internal/process
-# github.com/containers/storage v1.57.1
+# github.com/containers/storage v1.57.2-0.20250211190637-7aa96daee0a3
 ## explicit; go 1.22.0
 github.com/containers/storage
 github.com/containers/storage/drivers
@@ -1105,7 +1105,7 @@ github.com/ulikunitz/xz
 github.com/ulikunitz/xz/internal/hash
 github.com/ulikunitz/xz/internal/xlog
 github.com/ulikunitz/xz/lzma
-# github.com/vbatts/tar-split v0.11.7
+# github.com/vbatts/tar-split v0.12.1
 ## explicit; go 1.17
 github.com/vbatts/tar-split/archive/tar
 github.com/vbatts/tar-split/tar/asm