Merge pull request #10214 from containers/dependabot/go_modules/github.com/containers/image/v5-5.12.0

Bump github.com/containers/image/v5 from 5.11.1 to 5.12.0

go.mod

@@ -14,7 +14,7 @@ require (
 	github.com/containers/buildah v1.20.2-0.20210504130217-903dc56408ac
 	github.com/containers/common v0.37.2-0.20210503193405-42134aa138ce
 	github.com/containers/conmon v2.0.20+incompatible
-	github.com/containers/image/v5 v5.11.1
+	github.com/containers/image/v5 v5.12.0
 	github.com/containers/ocicrypt v1.1.1
 	github.com/containers/psgo v1.5.2
 	github.com/containers/storage v1.30.1

go.sum

@@ -198,8 +198,9 @@ github.com/containers/common v0.37.2-0.20210503193405-42134aa138ce h1:e7VNmGqwfU
 github.com/containers/common v0.37.2-0.20210503193405-42134aa138ce/go.mod h1:JjU+yvzIGyx8ZsY8nyf7snzs4VSNh1eIaYsqoSKBoRw=
 github.com/containers/conmon v2.0.20+incompatible h1:YbCVSFSCqFjjVwHTPINGdMX1F6JXHGTUje2ZYobNrkg=
 github.com/containers/conmon v2.0.20+incompatible/go.mod h1:hgwZ2mtuDrppv78a/cOBNiCm6O0UMWGx1mu7P00nu5I=
-github.com/containers/image/v5 v5.11.1 h1:mNybUvU6zXUwcMsQaa3n+Idsru5pV+GE7k4oRuPzYi0=
 github.com/containers/image/v5 v5.11.1/go.mod h1:HC9lhJ/Nz5v3w/5Co7H431kLlgzlVlOC+auD/er3OqE=
+github.com/containers/image/v5 v5.12.0 h1:1hNS2QkzFQ4lH3GYQLyAXB0acRMhS1Ubm6oV++8vw4w=
+github.com/containers/image/v5 v5.12.0/go.mod h1:VasTuHmOw+uD0oHCfApQcMO2+36SfyncoSahU7513Xs=
 github.com/containers/libtrust v0.0.0-20190913040956-14b96171aa3b h1:Q8ePgVfHDplZ7U33NwHZkrVELsZP5fYj9pM5WBZB2GE=
 github.com/containers/libtrust v0.0.0-20190913040956-14b96171aa3b/go.mod h1:9rfv8iPl1ZP7aqh9YA68wnZv2NUDbXdcdPHVz0pFbPY=
 github.com/containers/ocicrypt v1.0.1/go.mod h1:MeJDzk1RJHv89LjsH0Sp5KTY3ZYkjXO/C+bKAeWFIrc=

vendor/github.com/containers/image/v5/copy/copy.go

@@ -910,7 +910,7 @@ func (ic *imageCopier) copyLayers(ctx context.Context) error {
 	}
 
 	data := make([]copyLayerData, numLayers)
-	copyLayerHelper := func(index int, srcLayer types.BlobInfo, toEncrypt bool, pool *mpb.Progress) {
+	copyLayerHelper := func(index int, srcLayer types.BlobInfo, toEncrypt bool, pool *mpb.Progress, srcRef reference.Named) {
 		defer copySemaphore.Release(1)
 		defer copyGroup.Done()
 		cld := copyLayerData{}
@@ -925,7 +925,7 @@ func (ic *imageCopier) copyLayers(ctx context.Context) error {
 				logrus.Debugf("Skipping foreign layer %q copy to %s", cld.destInfo.Digest, ic.c.dest.Reference().Transport().Name())
 			}
 		} else {
-			cld.destInfo, cld.diffID, cld.err = ic.copyLayer(ctx, srcLayer, toEncrypt, pool, index)
+			cld.destInfo, cld.diffID, cld.err = ic.copyLayer(ctx, srcLayer, toEncrypt, pool, index, srcRef)
 		}
 		data[index] = cld
 	}
@@ -962,7 +962,7 @@ func (ic *imageCopier) copyLayers(ctx context.Context) error {
 				return errors.Wrapf(err, "Can't acquire semaphore")
 			}
 			copyGroup.Add(1)
-			go copyLayerHelper(i, srcLayer, encLayerBitmap[i], progressPool)
+			go copyLayerHelper(i, srcLayer, encLayerBitmap[i], progressPool, ic.c.rawSource.Reference().DockerReference())
 		}
 
 		// A call to copyGroup.Wait() is done at this point by the defer above.
@@ -1147,7 +1147,8 @@ type diffIDResult struct {
 
 // copyLayer copies a layer with srcInfo (with known Digest and Annotations and possibly known Size) in src to dest, perhaps (de/re/)compressing it,
 // and returns a complete blobInfo of the copied layer, and a value for LayerDiffIDs if diffIDIsNeeded
-func (ic *imageCopier) copyLayer(ctx context.Context, srcInfo types.BlobInfo, toEncrypt bool, pool *mpb.Progress, layerIndex int) (types.BlobInfo, digest.Digest, error) {
+// srcRef can be used as an additional hint to the destination during checking whehter a layer can be reused but srcRef can be nil.
+func (ic *imageCopier) copyLayer(ctx context.Context, srcInfo types.BlobInfo, toEncrypt bool, pool *mpb.Progress, layerIndex int, srcRef reference.Named) (types.BlobInfo, digest.Digest, error) {
 	// If the srcInfo doesn't contain compression information, try to compute it from the
 	// MediaType, which was either read from a manifest by way of LayerInfos() or constructed
 	// by LayerInfosForCopy(), if it was supplied at all.  If we succeed in copying the blob,
@@ -1189,11 +1190,14 @@ func (ic *imageCopier) copyLayer(ctx context.Context, srcInfo types.BlobInfo, to
 		// layers which requires passing the index of the layer.
 		// Hence, we need to special case and cast.
 		dest, ok := ic.c.dest.(internalTypes.ImageDestinationWithOptions)
-		if ok && enableEarlyCommit {
+		if ok {
 			options := internalTypes.TryReusingBlobOptions{
 				Cache:         ic.c.blobInfoCache,
 				CanSubstitute: ic.canSubstituteBlobs,
-				LayerIndex:    &layerIndex,
+				SrcRef:        srcRef,
+			}
+			if enableEarlyCommit {
+				options.LayerIndex = &layerIndex
 			}
 			reused, blobInfo, err = dest.TryReusingBlobWithOptions(ctx, srcInfo, options)
 		} else {
@@ -1550,12 +1554,12 @@ func (c *copier) copyBlobFromStream(ctx context.Context, srcStream io.Reader, sr
 	// which requires passing the index of the layer.  Hence, we need to
 	// special case and cast.
 	dest, ok := c.dest.(internalTypes.ImageDestinationWithOptions)
-	if ok && enableEarlyCommit {
+	if ok {
 		options := internalTypes.PutBlobOptions{
 			Cache:    c.blobInfoCache,
 			IsConfig: isConfig,
 		}
-		if !isConfig {
+		if !isConfig && enableEarlyCommit {
 			options.LayerIndex = &layerIndex
 		}
 		uploadedInfo, err = dest.PutBlobWithOptions(ctx, &errorAnnotationReader{destStream}, inputInfo, options)

vendor/github.com/containers/image/v5/internal/types/types.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"io"
 
+	"github.com/containers/image/v5/docker/reference"
 	publicTypes "github.com/containers/image/v5/types"
 )
 
@@ -50,4 +51,6 @@ type TryReusingBlobOptions struct {
 	CanSubstitute bool
 	// The corresponding index in the layer slice.
 	LayerIndex *int
+	// The reference of the image that contains the target blob.
+	SrcRef reference.Named
 }

vendor/github.com/containers/image/v5/storage/storage_image.go

@@ -76,11 +76,12 @@ type storageImageDestination struct {
 	indexToStorageID map[int]*string
 	// All accesses to below data are protected by `lock` which is made
 	// *explicit* in the code.
-	blobDiffIDs       map[digest.Digest]digest.Digest // Mapping from layer blobsums to their corresponding DiffIDs
+	blobDiffIDs         map[digest.Digest]digest.Digest           // Mapping from layer blobsums to their corresponding DiffIDs
-	fileSizes         map[digest.Digest]int64         // Mapping from layer blobsums to their sizes
+	fileSizes           map[digest.Digest]int64                   // Mapping from layer blobsums to their sizes
-	filenames         map[digest.Digest]string        // Mapping from layer blobsums to names of files we used to hold them
+	filenames           map[digest.Digest]string                  // Mapping from layer blobsums to names of files we used to hold them
-	currentIndex      int                             // The index of the layer to be committed (i.e., lower indices have already been committed)
+	currentIndex        int                                       // The index of the layer to be committed (i.e., lower indices have already been committed)
-	indexToPulledBlob map[int]*types.BlobInfo         // Mapping from layer (by index) to pulled down blob
+	indexToPulledBlob   map[int]*types.BlobInfo                   // Mapping from layer (by index) to pulled down blob
+	blobAdditionalLayer map[digest.Digest]storage.AdditionalLayer // Mapping from layer blobsums to their corresponding additional layer
 }
 
 type storageImageCloser struct {
@@ -391,16 +392,17 @@ func newImageDestination(sys *types.SystemContext, imageRef storageReference) (*
 		return nil, errors.Wrapf(err, "error creating a temporary directory")
 	}
 	image := &storageImageDestination{
-		imageRef:          imageRef,
+		imageRef:            imageRef,
-		directory:         directory,
+		directory:           directory,
-		signatureses:      make(map[digest.Digest][]byte),
+		signatureses:        make(map[digest.Digest][]byte),
-		blobDiffIDs:       make(map[digest.Digest]digest.Digest),
+		blobDiffIDs:         make(map[digest.Digest]digest.Digest),
-		fileSizes:         make(map[digest.Digest]int64),
+		blobAdditionalLayer: make(map[digest.Digest]storage.AdditionalLayer),
-		filenames:         make(map[digest.Digest]string),
+		fileSizes:           make(map[digest.Digest]int64),
-		SignatureSizes:    []int{},
+		filenames:           make(map[digest.Digest]string),
-		SignaturesSizes:   make(map[digest.Digest][]int),
+		SignatureSizes:      []int{},
-		indexToStorageID:  make(map[int]*string),
+		SignaturesSizes:     make(map[digest.Digest][]int),
-		indexToPulledBlob: make(map[int]*types.BlobInfo),
+		indexToStorageID:    make(map[int]*string),
+		indexToPulledBlob:   make(map[int]*types.BlobInfo),
 	}
 	return image, nil
 }
@@ -411,8 +413,11 @@ func (s *storageImageDestination) Reference() types.ImageReference {
 	return s.imageRef
 }
 
-// Close cleans up the temporary directory.
+// Close cleans up the temporary directory and additional layer store handlers.
 func (s *storageImageDestination) Close() error {
+	for _, al := range s.blobAdditionalLayer {
+		al.Release()
+	}
 	return os.RemoveAll(s.directory)
 }
 
@@ -532,7 +537,7 @@ func (s *storageImageDestination) PutBlob(ctx context.Context, stream io.Reader,
 // used the together.  Mixing the two with the non "WithOptions" functions
 // is not supported.
 func (s *storageImageDestination) TryReusingBlobWithOptions(ctx context.Context, blobinfo types.BlobInfo, options internalTypes.TryReusingBlobOptions) (bool, types.BlobInfo, error) {
-	reused, info, err := s.TryReusingBlob(ctx, blobinfo, options.Cache, options.CanSubstitute)
+	reused, info, err := s.tryReusingBlobWithSrcRef(ctx, blobinfo, options.Cache, options.CanSubstitute, options.SrcRef)
 	if err != nil || !reused || options.LayerIndex == nil {
 		return reused, info, err
 	}
@@ -540,6 +545,33 @@ func (s *storageImageDestination) TryReusingBlobWithOptions(ctx context.Context,
 	return reused, info, s.queueOrCommit(ctx, info, *options.LayerIndex)
 }
 
+// tryReusingBlobWithSrcRef is a wrapper around TryReusingBlob.
+// If ref is provided, this function first tries to get layer from Additional Layer Store.
+func (s *storageImageDestination) tryReusingBlobWithSrcRef(ctx context.Context, blobinfo types.BlobInfo, cache types.BlobInfoCache, canSubstitute bool, ref reference.Named) (bool, types.BlobInfo, error) {
+	// lock the entire method as it executes fairly quickly
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	if ref != nil {
+		// Check if we have the layer in the underlying additional layer store.
+		aLayer, err := s.imageRef.transport.store.LookupAdditionalLayer(blobinfo.Digest, ref.String())
+		if err != nil && errors.Cause(err) != storage.ErrLayerUnknown {
+			return false, types.BlobInfo{}, errors.Wrapf(err, `Error looking for compressed layers with digest %q and labels`, blobinfo.Digest)
+		} else if err == nil {
+			// Record the uncompressed value so that we can use it to calculate layer IDs.
+			s.blobDiffIDs[blobinfo.Digest] = aLayer.UncompressedDigest()
+			s.blobAdditionalLayer[blobinfo.Digest] = aLayer
+			return true, types.BlobInfo{
+				Digest:    blobinfo.Digest,
+				Size:      aLayer.CompressedSize(),
+				MediaType: blobinfo.MediaType,
+			}, nil
+		}
+	}
+
+	return s.tryReusingBlobLocked(ctx, blobinfo, cache, canSubstitute)
+}
+
 // TryReusingBlob checks whether the transport already contains, or can efficiently reuse, a blob, and if so, applies it to the current destination
 // (e.g. if the blob is a filesystem layer, this signifies that the changes it describes need to be applied again when composing a filesystem tree).
 // info.Digest must not be empty.
@@ -553,6 +585,13 @@ func (s *storageImageDestination) TryReusingBlob(ctx context.Context, blobinfo t
 	// lock the entire method as it executes fairly quickly
 	s.lock.Lock()
 	defer s.lock.Unlock()
+
+	return s.tryReusingBlobLocked(ctx, blobinfo, cache, canSubstitute)
+}
+
+// tryReusingBlobLocked implements a core functionality of TryReusingBlob.
+// This must be called with a lock being held on storageImageDestination.
+func (s *storageImageDestination) tryReusingBlobLocked(ctx context.Context, blobinfo types.BlobInfo, cache types.BlobInfoCache, canSubstitute bool) (bool, types.BlobInfo, error) {
 	if blobinfo.Digest == "" {
 		return false, types.BlobInfo{}, errors.Errorf(`Can not check for a blob with unknown digest`)
 	}
@@ -804,6 +843,20 @@ func (s *storageImageDestination) commitLayer(ctx context.Context, blob manifest
 		s.indexToStorageID[index] = &lastLayer
 		return nil
 	}
+
+	s.lock.Lock()
+	al, ok := s.blobAdditionalLayer[blob.Digest]
+	s.lock.Unlock()
+	if ok {
+		layer, err := al.PutAs(id, lastLayer, nil)
+		if err != nil {
+			return errors.Wrapf(err, "failed to put layer from digest and labels")
+		}
+		lastLayer = layer.ID
+		s.indexToStorageID[index] = &lastLayer
+		return nil
+	}
+
 	// Check if we previously cached a file with that blob's contents.  If we didn't,
 	// then we need to read the desired contents from a layer.
 	s.lock.Lock()

vendor/github.com/containers/image/v5/version/version.go

@@ -6,9 +6,9 @@ const (
 	// VersionMajor is for an API incompatible changes
 	VersionMajor = 5
 	// VersionMinor is for functionality in a backwards-compatible manner
-	VersionMinor = 11
+	VersionMinor = 12
 	// VersionPatch is for backwards-compatible bug fixes
-	VersionPatch = 1
+	VersionPatch = 0
 
 	// VersionDev indicates development branch. Releases will be empty string.
 	VersionDev = ""

vendor/modules.txt

@@ -122,7 +122,7 @@ github.com/containers/common/pkg/umask
 github.com/containers/common/version
 # github.com/containers/conmon v2.0.20+incompatible
 github.com/containers/conmon/runner/config
-# github.com/containers/image/v5 v5.11.1
+# github.com/containers/image/v5 v5.12.0
 github.com/containers/image/v5/copy
 github.com/containers/image/v5/directory
 github.com/containers/image/v5/directory/explicitfilepath