opensource/podman
1
0
Fork 0
mirror of https://github.com/containers/podman.git synced 2025-07-15 03:02:52 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #11028 from containers/dependabot/go_modules/github.com/containers/image/v5-5.14.0

Browse Source 
Bump github.com/containers/image/v5 from 5.13.2 to 5.14.0
This commit is contained in:
OpenShift Merge Robot
2021-07-27 15:26:24 +02:00
committed by GitHub
parent 508dc031c6 23a938fa26
commit 27a487929e
88 changed files with 2838 additions and 500 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
go.mod
View File

@ -14,7 +14,7 @@ require (
github.com/containers/buildah v1.21.1-0.20210721171232-54cafea4c933
github.com/containers/common v0.41.1-0.20210721172332-291287e9d060
github.com/containers/conmon v2.0.20+incompatible
github.com/containers/image/v5 v5.13.2
github.com/containers/image/v5 v5.14.0
github.com/containers/ocicrypt v1.1.2
github.com/containers/psgo v1.5.2
github.com/containers/storage v1.32.6
@ -64,7 +64,7 @@ require (
go.etcd.io/bbolt v1.3.6
golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
golang.org/x/sys v0.0.0-20210603125802-9665404d3644
golang.org/x/sys v0.0.0-20210616094352-59db8d763f22
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
k8s.io/api v0.21.3
k8s.io/apimachinery v0.21.3

9
go.sum
View File

@ -246,8 +246,9 @@ github.com/containers/common v0.41.1-0.20210721172332-291287e9d060 h1:HgGff2MeEK
github.com/containers/common v0.41.1-0.20210721172332-291287e9d060/go.mod h1:Ba5YVNCnyX6xDtg1JqEHa2EMVMW5UbHmIyEqsEwpeGE=
github.com/containers/conmon v2.0.20+incompatible h1:YbCVSFSCqFjjVwHTPINGdMX1F6JXHGTUje2ZYobNrkg=
github.com/containers/conmon v2.0.20+incompatible/go.mod h1:hgwZ2mtuDrppv78a/cOBNiCm6O0UMWGx1mu7P00nu5I=
github.com/containers/image/v5 v5.13.2 h1:AgYunV/9d2fRkrmo23wH2MkqeHolFd6oQCkK+1PpuFA=
github.com/containers/image/v5 v5.13.2/go.mod h1:GkWursKDlDcUIT7L7vZf70tADvZCk/Ga0wgS0MuF0ag=
github.com/containers/image/v5 v5.14.0 h1:ORaFZ/NwFjkSunMhxg9I8fQod8pgXkrYNiZzit/VqOE=
github.com/containers/image/v5 v5.14.0/go.mod h1:SxiBKOcKuT+4yTjD0AskjO+UwFvNcVOJ9qlAw1HNSPU=
github.com/containers/libtrust v0.0.0-20190913040956-14b96171aa3b h1:Q8ePgVfHDplZ7U33NwHZkrVELsZP5fYj9pM5WBZB2GE=
github.com/containers/libtrust v0.0.0-20190913040956-14b96171aa3b/go.mod h1:9rfv8iPl1ZP7aqh9YA68wnZv2NUDbXdcdPHVz0pFbPY=
github.com/containers/ocicrypt v1.0.1/go.mod h1:MeJDzk1RJHv89LjsH0Sp5KTY3ZYkjXO/C+bKAeWFIrc=
@ -896,8 +897,9 @@ github.com/vbatts/tar-split v0.11.1 h1:0Odu65rhcZ3JZaPHxl7tCI3V/C/Q9Zf82UFravl02
github.com/vbatts/tar-split v0.11.1/go.mod h1:LEuURwDEiWjRjwu46yU3KVGuUdVv/dcnpcEPSzR8z6g=
github.com/vbauerster/mpb/v6 v6.0.4 h1:h6J5zM/2wimP5Hj00unQuV8qbo5EPcj6wbkCqgj7KcY=
github.com/vbauerster/mpb/v6 v6.0.4/go.mod h1:a/+JT57gqh6Du0Ay5jSR+uBMfXGdlR7VQlGP52fJxLM=
github.com/vbauerster/mpb/v7 v7.0.2 h1:eN6AD/ytv1nqCO7Dm8MO0/pGMKmMyH/WMnTJhAUuc/w=
github.com/vbauerster/mpb/v7 v7.0.2/go.mod h1:Mnq3gESXJ9eQhccbGZDggJ1faTCrmaA4iN57fUloRGE=
github.com/vbauerster/mpb/v7 v7.0.3 h1:NfX0pHWhlDTev15M/C3qmSTM1EiIjcS+/d6qS6H4FnI=
github.com/vbauerster/mpb/v7 v7.0.3/go.mod h1:NXGsfPGx6G2JssqvEcULtDqUrxuuYs4llpv8W6ZUpzk=
github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk=
github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE=
github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852 h1:cPXZWzzG0NllBLdjWoD1nDfaqu98YMv+OneaKc8sPOA=
@ -1177,8 +1179,9 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210603125802-9665404d3644 h1:CA1DEQ4NdKphKeL70tvsWNdT5oFh1lOjihRcEDROi0I=
golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210616094352-59db8d763f22 h1:RqytpXGR1iVNX7psjB3ff8y7sNFinVFvkx1c8SjBkio=
golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201113234701-d7a72108b828/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=

2
test/e2e/pull_test.go
View File

@ -44,7 +44,7 @@ var _ = Describe("Podman pull", func() {
session = podmanTest.Podman([]string{"pull", "busybox:latest", "docker.io/library/ibetthisdoesnotexistfr:random", "alpine"})
session.WaitWithDefaultTimeout()
Expect(session).Should(Exit(125))
expectedError := "Error initializing source docker://ibetthisdoesnotexistfr:random"
expectedError := "initializing source docker://ibetthisdoesnotexistfr:random"
found, _ := session.ErrorGrepString(expectedError)
Expect(found).To(Equal(true))

263
vendor/github.com/containers/image/v5/copy/copy.go generated vendored
View File

@ -92,7 +92,7 @@ func (d *digestingReader) Read(p []byte) (int, error) {
// Coverage: This should not happen, the hash.Hash interface requires
// d.digest.Write to never return an error, and the io.Writer interface
// requires n2 == len(input) if no error is returned.
return 0, errors.Wrapf(err, "Error updating digest during verification: %d vs. %d", n2, n)
return 0, errors.Wrapf(err, "updating digest during verification: %d vs. %d", n2, n)
}
}
if err == io.EOF {
@ -123,6 +123,7 @@ type copier struct {
ociEncryptConfig *encconfig.EncryptConfig
maxParallelDownloads uint
downloadForeignLayers bool
fetchPartialBlobs bool
}
// imageCopier tracks state specific to a single image (possibly an item of a manifest list)
@ -194,15 +195,21 @@ type Options struct {
// OciDecryptConfig contains the config that can be used to decrypt an image if it is
// encrypted if non-nil. If nil, it does not attempt to decrypt an image.
OciDecryptConfig *encconfig.DecryptConfig
// MaxParallelDownloads indicates the maximum layers to pull at the same time. A reasonable default is used if this is left as 0.
MaxParallelDownloads uint
// When OptimizeDestinationImageAlreadyExists is set, optimize the copy assuming that the destination image already
// exists (and is equivalent). Making the eventual (no-op) copy more performant for this case. Enabling the option
// is slightly pessimistic if the destination image doesn't exist, or is not equivalent.
OptimizeDestinationImageAlreadyExists bool
// Download layer contents with "nondistributable" media types ("foreign" layers) and translate the layer media type
// to not indicate "nondistributable".
DownloadForeignLayers bool
// FetchPartialBlobs indicates whether to attempt to fetch the blob partially. Experimental.
FetchPartialBlobs bool
}
// validateImageListSelection returns an error if the passed-in value is not one that we recognize as a valid ImageListSelection value
@ -240,7 +247,7 @@ func Image(ctx context.Context, policyContext *signature.PolicyContext, destRef,
dest, err := destRef.NewImageDestination(ctx, options.DestinationCtx)
if err != nil {
return nil, errors.Wrapf(err, "Error initializing destination %s", transports.ImageName(destRef))
return nil, errors.Wrapf(err, "initializing destination %s", transports.ImageName(destRef))
}
defer func() {
if err := dest.Close(); err != nil {
@ -250,7 +257,7 @@ func Image(ctx context.Context, policyContext *signature.PolicyContext, destRef,
rawSource, err := srcRef.NewImageSource(ctx, options.SourceCtx)
if err != nil {
return nil, errors.Wrapf(err, "Error initializing source %s", transports.ImageName(srcRef))
return nil, errors.Wrapf(err, "initializing source %s", transports.ImageName(srcRef))
}
defer func() {
if err := rawSource.Close(); err != nil {
@ -283,6 +290,7 @@ func Image(ctx context.Context, policyContext *signature.PolicyContext, destRef,
ociEncryptConfig: options.OciEncryptConfig,
maxParallelDownloads: options.MaxParallelDownloads,
downloadForeignLayers: options.DownloadForeignLayers,
fetchPartialBlobs: options.FetchPartialBlobs,
}
// Default to using gzip compression unless specified otherwise.
if options.DestinationCtx == nil || options.DestinationCtx.CompressionFormat == nil {
@ -302,7 +310,7 @@ func Image(ctx context.Context, policyContext *signature.PolicyContext, destRef,
unparsedToplevel := image.UnparsedInstance(rawSource, nil)
multiImage, err := isMultiImage(ctx, unparsedToplevel)
if err != nil {
return nil, errors.Wrapf(err, "Error determining manifest MIME type for %s", transports.ImageName(srcRef))
return nil, errors.Wrapf(err, "determining manifest MIME type for %s", transports.ImageName(srcRef))
}
if !multiImage {
@ -315,15 +323,15 @@ func Image(ctx context.Context, policyContext *signature.PolicyContext, destRef,
// matches the current system to copy, and copy it.
mfest, manifestType, err := unparsedToplevel.Manifest(ctx)
if err != nil {
return nil, errors.Wrapf(err, "Error reading manifest for %s", transports.ImageName(srcRef))
return nil, errors.Wrapf(err, "reading manifest for %s", transports.ImageName(srcRef))
}
manifestList, err := manifest.ListFromBlob(mfest, manifestType)
if err != nil {
return nil, errors.Wrapf(err, "Error parsing primary manifest as list for %s", transports.ImageName(srcRef))
return nil, errors.Wrapf(err, "parsing primary manifest as list for %s", transports.ImageName(srcRef))
}
instanceDigest, err := manifestList.ChooseInstance(options.SourceCtx) // try to pick one that matches options.SourceCtx
if err != nil {
return nil, errors.Wrapf(err, "Error choosing an image from manifest list %s", transports.ImageName(srcRef))
return nil, errors.Wrapf(err, "choosing an image from manifest list %s", transports.ImageName(srcRef))
}
logrus.Debugf("Source is a manifest list; copying (only) instance %s for current system", instanceDigest)
unparsedInstance := image.UnparsedInstance(rawSource, &instanceDigest)
@ -334,7 +342,7 @@ func Image(ctx context.Context, policyContext *signature.PolicyContext, destRef,
} else { /* options.ImageListSelection == CopyAllImages or options.ImageListSelection == CopySpecificImages, */
// If we were asked to copy multiple images and can't, that's an error.
if !supportsMultipleImages(c.dest) {
return nil, errors.Errorf("Error copying multiple images: destination transport %q does not support copying multiple images as a group", destRef.Transport().Name())
return nil, errors.Errorf("copying multiple images: destination transport %q does not support copying multiple images as a group", destRef.Transport().Name())
}
// Copy some or all of the images.
switch options.ImageListSelection {
@ -343,13 +351,13 @@ func Image(ctx context.Context, policyContext *signature.PolicyContext, destRef,
case CopySpecificImages:
logrus.Debugf("Source is a manifest list; copying some instances")
}
if copiedManifest, _, err = c.copyMultipleImages(ctx, policyContext, options, unparsedToplevel); err != nil {
if copiedManifest, err = c.copyMultipleImages(ctx, policyContext, options, unparsedToplevel); err != nil {
return nil, err
}
}
if err := c.dest.Commit(ctx, unparsedToplevel); err != nil {
return nil, errors.Wrap(err, "Error committing the finished image")
return nil, errors.Wrap(err, "committing the finished image")
}
return copiedManifest, nil
@ -376,12 +384,12 @@ func supportsMultipleImages(dest types.ImageDestination) bool {
func compareImageDestinationManifestEqual(ctx context.Context, options *Options, src types.Image, targetInstance *digest.Digest, dest types.ImageDestination) (bool, []byte, string, digest.Digest, error) {
srcManifest, _, err := src.Manifest(ctx)
if err != nil {
return false, nil, "", "", errors.Wrapf(err, "Error reading manifest from image")
return false, nil, "", "", errors.Wrapf(err, "reading manifest from image")
}
srcManifestDigest, err := manifest.Digest(srcManifest)
if err != nil {
return false, nil, "", "", errors.Wrapf(err, "Error calculating manifest digest")
return false, nil, "", "", errors.Wrapf(err, "calculating manifest digest")
}
destImageSource, err := dest.Reference().NewImageSource(ctx, options.DestinationCtx)
@ -398,7 +406,7 @@ func compareImageDestinationManifestEqual(ctx context.Context, options *Options,
destManifestDigest, err := manifest.Digest(destManifest)
if err != nil {
return false, nil, "", "", errors.Wrapf(err, "Error calculating manifest digest")
return false, nil, "", "", errors.Wrapf(err, "calculating manifest digest")
}
logrus.Debugf("Comparing source and destination manifest digests: %v vs. %v", srcManifestDigest, destManifestDigest)
@ -412,15 +420,15 @@ func compareImageDestinationManifestEqual(ctx context.Context, options *Options,
// copyMultipleImages copies some or all of an image list's instances, using
// policyContext to validate source image admissibility.
func (c *copier) copyMultipleImages(ctx context.Context, policyContext *signature.PolicyContext, options *Options, unparsedToplevel *image.UnparsedImage) (copiedManifest []byte, copiedManifestType string, retErr error) {
func (c *copier) copyMultipleImages(ctx context.Context, policyContext *signature.PolicyContext, options *Options, unparsedToplevel *image.UnparsedImage) (copiedManifest []byte, retErr error) {
// Parse the list and get a copy of the original value after it's re-encoded.
manifestList, manifestType, err := unparsedToplevel.Manifest(ctx)
if err != nil {
return nil, "", errors.Wrapf(err, "Error reading manifest list")
return nil, errors.Wrapf(err, "reading manifest list")
}
originalList, err := manifest.ListFromBlob(manifestList, manifestType)
if err != nil {
return nil, "", errors.Wrapf(err, "Error parsing manifest list %q", string(manifestList))
return nil, errors.Wrapf(err, "parsing manifest list %q", string(manifestList))
}
updatedList := originalList.Clone()
@ -432,14 +440,14 @@ func (c *copier) copyMultipleImages(ctx context.Context, policyContext *signatur
c.Printf("Getting image list signatures\n")
s, err := c.rawSource.GetSignatures(ctx, nil)
if err != nil {
return nil, "", errors.Wrap(err, "Error reading signatures")
return nil, errors.Wrap(err, "reading signatures")
}
sigs = s
}
if len(sigs) != 0 {
c.Printf("Checking if image list destination supports signatures\n")
if err := c.dest.SupportsSignatures(ctx); err != nil {
return nil, "", errors.Wrapf(err, "Can not copy signatures to %s", transports.ImageName(c.dest.Reference()))
return nil, errors.Wrapf(err, "Can not copy signatures to %s", transports.ImageName(c.dest.Reference()))
}
}
canModifyManifestList := (len(sigs) == 0)
@ -454,11 +462,11 @@ func (c *copier) copyMultipleImages(ctx context.Context, policyContext *signatur
}
selectedListType, otherManifestMIMETypeCandidates, err := c.determineListConversion(manifestType, c.dest.SupportedManifestMIMETypes(), forceListMIMEType)
if err != nil {
return nil, "", errors.Wrapf(err, "Error determining manifest list type to write to destination")
return nil, errors.Wrapf(err, "determining manifest list type to write to destination")
}
if selectedListType != originalList.MIMEType() {
if !canModifyManifestList {
return nil, "", errors.Errorf("Error: manifest list must be converted to type %q to be written to destination, but that would invalidate signatures", selectedListType)
return nil, errors.Errorf("manifest list must be converted to type %q to be written to destination, but that would invalidate signatures", selectedListType)
}
}
@ -483,7 +491,7 @@ func (c *copier) copyMultipleImages(ctx context.Context, policyContext *signatur
if skip {
update, err := updatedList.Instance(instanceDigest)
if err != nil {
return nil, "", err
return nil, err
}
logrus.Debugf("Skipping instance %s (%d/%d)", instanceDigest, i+1, len(instanceDigests))
// Record the digest/size/type of the manifest that we didn't copy.
@ -496,7 +504,7 @@ func (c *copier) copyMultipleImages(ctx context.Context, policyContext *signatur
unparsedInstance := image.UnparsedInstance(c.rawSource, &instanceDigest)
updatedManifest, updatedManifestType, updatedManifestDigest, err := c.copyOneImage(ctx, policyContext, options, unparsedToplevel, unparsedInstance, &instanceDigest)
if err != nil {
return nil, "", err
return nil, err
}
instancesCopied++
// Record the result of a possible conversion here.
@ -510,7 +518,7 @@ func (c *copier) copyMultipleImages(ctx context.Context, policyContext *signatur
// Now reset the digest/size/types of the manifests in the list to account for any conversions that we made.
if err = updatedList.UpdateInstances(updates); err != nil {
return nil, "", errors.Wrapf(err, "Error updating manifest list")
return nil, errors.Wrapf(err, "updating manifest list")
}
// Iterate through supported list types, preferred format first.
@ -525,7 +533,7 @@ func (c *copier) copyMultipleImages(ctx context.Context, policyContext *signatur
if thisListType != updatedList.MIMEType() {
attemptedList, err = updatedList.ConvertToMIMEType(thisListType)
if err != nil {
return nil, "", errors.Wrapf(err, "Error converting manifest list to list with MIME type %q", thisListType)
return nil, errors.Wrapf(err, "converting manifest list to list with MIME type %q", thisListType)
}
}
@ -533,17 +541,17 @@ func (c *copier) copyMultipleImages(ctx context.Context, policyContext *signatur
// by serializing them both so that we can compare them.
attemptedManifestList, err := attemptedList.Serialize()
if err != nil {
return nil, "", errors.Wrapf(err, "Error encoding updated manifest list (%q: %#v)", updatedList.MIMEType(), updatedList.Instances())
return nil, errors.Wrapf(err, "encoding updated manifest list (%q: %#v)", updatedList.MIMEType(), updatedList.Instances())
}
originalManifestList, err := originalList.Serialize()
if err != nil {
return nil, "", errors.Wrapf(err, "Error encoding original manifest list for comparison (%q: %#v)", originalList.MIMEType(), originalList.Instances())
return nil, errors.Wrapf(err, "encoding original manifest list for comparison (%q: %#v)", originalList.MIMEType(), originalList.Instances())
}
// If we can't just use the original value, but we have to change it, flag an error.
if !bytes.Equal(attemptedManifestList, originalManifestList) {
if !canModifyManifestList {
return nil, "", errors.Errorf("Error: manifest list must be converted to type %q to be written to destination, but that would invalidate signatures", thisListType)
return nil, errors.Errorf(" manifest list must be converted to type %q to be written to destination, but that would invalidate signatures", thisListType)
}
logrus.Debugf("Manifest list has been updated")
} else {
@ -563,24 +571,24 @@ func (c *copier) copyMultipleImages(ctx context.Context, policyContext *signatur
break
}
if errs != nil {
return nil, "", fmt.Errorf("Uploading manifest list failed, attempted the following formats: %s", strings.Join(errs, ", "))
return nil, fmt.Errorf("Uploading manifest list failed, attempted the following formats: %s", strings.Join(errs, ", "))
}
// Sign the manifest list.
if options.SignBy != "" {
newSig, err := c.createSignature(manifestList, options.SignBy)
if err != nil {
return nil, "", err
return nil, err
}
sigs = append(sigs, newSig)
}
c.Printf("Storing list signatures\n")
if err := c.dest.PutSignatures(ctx, sigs, nil); err != nil {
return nil, "", errors.Wrap(err, "Error writing signatures")
return nil, errors.Wrap(err, "writing signatures")
}
return manifestList, selectedListType, nil
return manifestList, nil
}
// copyOneImage copies a single (non-manifest-list) image unparsedImage, using policyContext to validate
@ -591,7 +599,7 @@ func (c *copier) copyOneImage(ctx context.Context, policyContext *signature.Poli
multiImage, err := isMultiImage(ctx, unparsedImage)
if err != nil {
// FIXME FIXME: How to name a reference for the sub-image?
return nil, "", "", errors.Wrapf(err, "Error determining manifest MIME type for %s", transports.ImageName(unparsedImage.Reference()))
return nil, "", "", errors.Wrapf(err, "determining manifest MIME type for %s", transports.ImageName(unparsedImage.Reference()))
}
if multiImage {
return nil, "", "", fmt.Errorf("Unexpectedly received a manifest list instead of a manifest for a single image")
@ -605,7 +613,7 @@ func (c *copier) copyOneImage(ctx context.Context, policyContext *signature.Poli
}
src, err := image.FromUnparsedImage(ctx, options.SourceCtx, unparsedImage)
if err != nil {
return nil, "", "", errors.Wrapf(err, "Error initializing image from source %s", transports.ImageName(c.rawSource.Reference()))
return nil, "", "", errors.Wrapf(err, "initializing image from source %s", transports.ImageName(c.rawSource.Reference()))
}
// If the destination is a digested reference, make a note of that, determine what digest value we're
@ -617,20 +625,20 @@ func (c *copier) copyOneImage(ctx context.Context, policyContext *signature.Poli
destIsDigestedReference = true
sourceManifest, _, err := src.Manifest(ctx)
if err != nil {
return nil, "", "", errors.Wrapf(err, "Error reading manifest from source image")
return nil, "", "", errors.Wrapf(err, "reading manifest from source image")
}
matches, err := manifest.MatchesDigest(sourceManifest, digested.Digest())
if err != nil {
return nil, "", "", errors.Wrapf(err, "Error computing digest of source image's manifest")
return nil, "", "", errors.Wrapf(err, "computing digest of source image's manifest")
}
if !matches {
manifestList, _, err := unparsedToplevel.Manifest(ctx)
if err != nil {
return nil, "", "", errors.Wrapf(err, "Error reading manifest from source image")
return nil, "", "", errors.Wrapf(err, "reading manifest from source image")
}
matches, err = manifest.MatchesDigest(manifestList, digested.Digest())
if err != nil {
return nil, "", "", errors.Wrapf(err, "Error computing digest of source image's manifest")
return nil, "", "", errors.Wrapf(err, "computing digest of source image's manifest")
}
if !matches {
return nil, "", "", errors.New("Digest of source image's manifest would not match destination reference")
@ -650,7 +658,7 @@ func (c *copier) copyOneImage(ctx context.Context, policyContext *signature.Poli
c.Printf("Getting image source signatures\n")
s, err := src.Signatures(ctx)
if err != nil {
return nil, "", "", errors.Wrap(err, "Error reading signatures")
return nil, "", "", errors.Wrap(err, "reading signatures")
}
sigs = s
}
@ -785,7 +793,7 @@ func (c *copier) copyOneImage(ctx context.Context, policyContext *signature.Poli
c.Printf("Storing signatures\n")
if err := c.dest.PutSignatures(ctx, sigs, targetInstance); err != nil {
return nil, "", "", errors.Wrap(err, "Error writing signatures")
return nil, "", "", errors.Wrap(err, "writing signatures")
}
return manifestBytes, retManifestType, retManifestDigest, nil
@ -805,11 +813,11 @@ func checkImageDestinationForCurrentRuntime(ctx context.Context, sys *types.Syst
if dest.MustMatchRuntimeOS() {
c, err := src.OCIConfig(ctx)
if err != nil {
return errors.Wrapf(err, "Error parsing image configuration")
return errors.Wrapf(err, "parsing image configuration")
}
wantedPlatforms, err := platform.WantedPlatforms(sys)
if err != nil {
return errors.Wrapf(err, "error getting current platform information %#v", sys)
return errors.Wrapf(err, "getting current platform information %#v", sys)
}
options := newOrderedSet()
@ -1034,13 +1042,13 @@ func (ic *imageCopier) copyUpdatedConfigAndManifest(ctx context.Context, instanc
}
pi, err := ic.src.UpdatedImage(ctx, *ic.manifestUpdates)
if err != nil {
return nil, "", errors.Wrap(err, "Error creating an updated image manifest")
return nil, "", errors.Wrap(err, "creating an updated image manifest")
}
pendingImage = pi
}
man, _, err := pendingImage.Manifest(ctx)
if err != nil {
return nil, "", errors.Wrap(err, "Error reading manifest")
return nil, "", errors.Wrap(err, "reading manifest")
}
if err := ic.c.copyConfig(ctx, pendingImage); err != nil {
@ -1056,7 +1064,7 @@ func (ic *imageCopier) copyUpdatedConfigAndManifest(ctx context.Context, instanc
instanceDigest = &manifestDigest
}
if err := ic.c.dest.PutManifest(ctx, man, instanceDigest); err != nil {
return nil, "", errors.Wrapf(err, "Error writing manifest %q", string(man))
return nil, "", errors.Wrapf(err, "writing manifest %q", string(man))
}
return man, manifestDigest, nil
}
@ -1072,9 +1080,25 @@ func (c *copier) newProgressPool(ctx context.Context) (*mpb.Progress, func()) {
}
}
// customPartialBlobCounter provides a decorator function for the partial blobs retrieval progress bar
func customPartialBlobCounter(filler interface{}, wcc ...decor.WC) decor.Decorator {
producer := func(filler interface{}) decor.DecorFunc {
return func(s decor.Statistics) string {
if s.Total == 0 {
pairFmt := "%.1f / %.1f (skipped: %.1f)"
return fmt.Sprintf(pairFmt, decor.SizeB1024(s.Current), decor.SizeB1024(s.Total), decor.SizeB1024(s.Refill))
}
pairFmt := "%.1f / %.1f (skipped: %.1f = %.2f%%)"
percentage := 100.0 * float64(s.Refill) / float64(s.Total)
return fmt.Sprintf(pairFmt, decor.SizeB1024(s.Current), decor.SizeB1024(s.Total), decor.SizeB1024(s.Refill), percentage)
}
}
return decor.Any(producer(filler), wcc...)
}
// createProgressBar creates a mpb.Bar in pool. Note that if the copier's reportWriter
// is ioutil.Discard, the progress bar's output will be discarded
func (c *copier) createProgressBar(pool *mpb.Progress, info types.BlobInfo, kind string, onComplete string) *mpb.Bar {
func (c *copier) createProgressBar(pool *mpb.Progress, partial bool, info types.BlobInfo, kind string, onComplete string) *mpb.Bar {
// shortDigestLen is the length of the digest used for blobs.
const shortDigestLen = 12
@ -1091,18 +1115,30 @@ func (c *copier) createProgressBar(pool *mpb.Progress, info types.BlobInfo, kind
// Use a normal progress bar when we know the size (i.e., size > 0).
// Otherwise, use a spinner to indicate that something's happening.
var bar *mpb.Bar
sstyle := mpb.SpinnerStyle(".", "..", "...", "....", "").PositionLeft()
if info.Size > 0 {
bar = pool.AddBar(info.Size,
mpb.BarFillerClearOnComplete(),
mpb.PrependDecorators(
decor.OnComplete(decor.Name(prefix), onComplete),
),
mpb.AppendDecorators(
decor.OnComplete(decor.CountersKibiByte("%.1f / %.1f"), ""),
),
)
if partial {
bar = pool.AddBar(info.Size,
mpb.BarFillerClearOnComplete(),
mpb.PrependDecorators(
decor.OnComplete(decor.Name(prefix), onComplete),
),
mpb.AppendDecorators(
customPartialBlobCounter(sstyle.Build()),
),
)
} else {
bar = pool.AddBar(info.Size,
mpb.BarFillerClearOnComplete(),
mpb.PrependDecorators(
decor.OnComplete(decor.Name(prefix), onComplete),
),
mpb.AppendDecorators(
decor.OnComplete(decor.CountersKibiByte("%.1f / %.1f"), ""),
),
)
}
} else {
sstyle := mpb.SpinnerStyle(".", "..", "...", "....", "").PositionLeft()
bar = pool.Add(0,
sstyle.Build(),
mpb.BarFillerClearOnComplete(),
@ -1123,13 +1159,13 @@ func (c *copier) copyConfig(ctx context.Context, src types.Image) error {
if srcInfo.Digest != "" {
configBlob, err := src.ConfigBlob(ctx)
if err != nil {
return errors.Wrapf(err, "Error reading config blob %s", srcInfo.Digest)
return errors.Wrapf(err, "reading config blob %s", srcInfo.Digest)
}
destInfo, err := func() (types.BlobInfo, error) { // A scope for defer
progressPool, progressCleanup := c.newProgressPool(ctx)
defer progressCleanup()
bar := c.createProgressBar(progressPool, srcInfo, "config", "done")
bar := c.createProgressBar(progressPool, false, srcInfo, "config", "done")
destInfo, err := c.copyBlobFromStream(ctx, bytes.NewReader(configBlob), srcInfo, nil, false, true, false, bar, -1, false)
if err != nil {
return types.BlobInfo{}, err
@ -1213,11 +1249,11 @@ func (ic *imageCopier) copyLayer(ctx context.Context, srcInfo types.BlobInfo, to
}
if err != nil {
return types.BlobInfo{}, "", errors.Wrapf(err, "Error trying to reuse blob %s at destination", srcInfo.Digest)
return types.BlobInfo{}, "", errors.Wrapf(err, "trying to reuse blob %s at destination", srcInfo.Digest)
}
if reused {
logrus.Debugf("Skipping blob %s (already present):", srcInfo.Digest)
bar := ic.c.createProgressBar(pool, srcInfo, "blob", "skipped: already exists")
bar := ic.c.createProgressBar(pool, false, srcInfo, "blob", "skipped: already exists")
bar.SetTotal(0, true)
// Throw an event that the layer has been skipped
@ -1244,14 +1280,57 @@ func (ic *imageCopier) copyLayer(ctx context.Context, srcInfo types.BlobInfo, to
}
}
// A partial pull is managed by the destination storage, that decides what portions
// of the source file are not known yet and must be fetched.
// Attempt a partial only when the source allows to retrieve a blob partially and
// the destination has support for it.
imgSource, okSource := ic.c.rawSource.(internalTypes.ImageSourceSeekable)
imgDest, okDest := ic.c.dest.(internalTypes.ImageDestinationPartial)
if ic.c.fetchPartialBlobs && okSource && okDest && !diffIDIsNeeded {
bar := ic.c.createProgressBar(pool, true, srcInfo, "blob", "done")
progress := make(chan int64)
terminate := make(chan interface{})
defer close(terminate)
defer close(progress)
proxy := imageSourceSeekableProxy{
source: imgSource,
progress: progress,
}
go func() {
for {
select {
case written := <-progress:
bar.IncrInt64(written)
case <-terminate:
return
}
}
}()
bar.SetTotal(srcInfo.Size, false)
info, err := imgDest.PutBlobPartial(ctx, proxy, srcInfo, ic.c.blobInfoCache)
if err == nil {
bar.SetRefill(srcInfo.Size - bar.Current())
bar.SetCurrent(srcInfo.Size)
bar.SetTotal(srcInfo.Size, true)
logrus.Debugf("Retrieved partial blob %v", srcInfo.Digest)
return info, cachedDiffID, nil
}
bar.Abort(true)
logrus.Errorf("Failed to retrieve partial blob: %v", err)
}
// Fallback: copy the layer, computing the diffID if we need to do so
srcStream, srcBlobSize, err := ic.c.rawSource.GetBlob(ctx, srcInfo, ic.c.blobInfoCache)
if err != nil {
return types.BlobInfo{}, "", errors.Wrapf(err, "Error reading blob %s", srcInfo.Digest)
return types.BlobInfo{}, "", errors.Wrapf(err, "reading blob %s", srcInfo.Digest)
}
defer srcStream.Close()
bar := ic.c.createProgressBar(pool, srcInfo, "blob", "done")
bar := ic.c.createProgressBar(pool, false, srcInfo, "blob", "done")
blobInfo, diffIDChan, err := ic.copyLayerFromStream(ctx, srcStream, types.BlobInfo{Digest: srcInfo.Digest, Size: srcBlobSize, MediaType: srcInfo.MediaType, Annotations: srcInfo.Annotations}, diffIDIsNeeded, toEncrypt, bar, layerIndex, emptyLayer)
if err != nil {
@ -1265,7 +1344,7 @@ func (ic *imageCopier) copyLayer(ctx context.Context, srcInfo types.BlobInfo, to
return types.BlobInfo{}, "", ctx.Err()
case diffIDResult := <-diffIDChan:
if diffIDResult.err != nil {
return types.BlobInfo{}, "", errors.Wrap(diffIDResult.err, "Error computing layer DiffID")
return types.BlobInfo{}, "", errors.Wrap(diffIDResult.err, "computing layer DiffID")
}
logrus.Debugf("Computed DiffID %s for layer %s", diffIDResult.digest, srcInfo.Digest)
// This is safe because we have just computed diffIDResult.Digest ourselves, and in the process
@ -1288,7 +1367,7 @@ func (ic *imageCopier) copyLayerFromStream(ctx context.Context, srcStream io.Rea
var getDiffIDRecorder func(compression.DecompressorFunc) io.Writer // = nil
var diffIDChan chan diffIDResult
err := errors.New("Internal error: unexpected panic in copyLayer") // For pipeWriter.CloseWithError below
err := errors.New("Internal error: unexpected panic in copyLayer") // For pipeWriter.CloseWithbelow
if diffIDIsNeeded {
diffIDChan = make(chan diffIDResult, 1) // Buffered, so that sending a value after this or our caller has failed and exited does not block.
pipeReader, pipeWriter := io.Pipe()
@ -1350,7 +1429,7 @@ type errorAnnotationReader struct {
func (r errorAnnotationReader) Read(b []byte) (n int, err error) {
n, err = r.reader.Read(b)
if err != io.EOF {
return n, errors.Wrapf(err, "error happened during read")
return n, errors.Wrapf(err, "happened during read")
}
return n, err
}
@ -1377,7 +1456,7 @@ func (c *copier) copyBlobFromStream(ctx context.Context, srcStream io.Reader, sr
// read stream to the end, and validation does not happen.
digestingReader, err := newDigestingReader(srcStream, srcInfo.Digest)
if err != nil {
return types.BlobInfo{}, errors.Wrapf(err, "Error preparing to verify blob %s", srcInfo.Digest)
return types.BlobInfo{}, errors.Wrapf(err, "preparing to verify blob %s", srcInfo.Digest)
}
var destStream io.Reader = digestingReader
@ -1391,7 +1470,7 @@ func (c *copier) copyBlobFromStream(ctx context.Context, srcStream io.Reader, sr
var d digest.Digest
destStream, d, err = ocicrypt.DecryptLayer(c.ociDecryptConfig, destStream, newDesc, false)
if err != nil {
return types.BlobInfo{}, errors.Wrapf(err, "Error decrypting layer %s", srcInfo.Digest)
return types.BlobInfo{}, errors.Wrapf(err, "decrypting layer %s", srcInfo.Digest)
}
srcInfo.Digest = d
@ -1408,7 +1487,7 @@ func (c *copier) copyBlobFromStream(ctx context.Context, srcStream io.Reader, sr
// This requires us to “peek ahead” into the stream to read the initial part, which requires us to chain through another io.Reader returned by DetectCompression.
compressionFormat, decompressor, destStream, err := compression.DetectCompressionFormat(destStream) // We could skip this in some cases, but let's keep the code path uniform
if err != nil {
return types.BlobInfo{}, errors.Wrapf(err, "Error reading blob %s", srcInfo.Digest)
return types.BlobInfo{}, errors.Wrapf(err, "reading blob %s", srcInfo.Digest)
}
isCompressed := decompressor != nil
if expectedCompressionFormat, known := expectedCompressionFormats[srcInfo.MediaType]; known && isCompressed && compressionFormat.Name() != expectedCompressionFormat.Name() {
@ -1425,6 +1504,7 @@ func (c *copier) copyBlobFromStream(ctx context.Context, srcStream io.Reader, sr
originalLayerReader = destStream
}
compressionMetadata := map[string]string{}
// === Deal with layer compression/decompression if necessary
// WARNING: If you are adding new reasons to change the blob, update also the OptimizeDestinationImageAlreadyExists
// short-circuit conditions
@ -1453,7 +1533,7 @@ func (c *copier) copyBlobFromStream(ctx context.Context, srcStream io.Reader, sr
// If this fails while writing data, it will do pipeWriter.CloseWithError(); if it fails otherwise,
// e.g. because we have exited and due to pipeReader.Close() above further writing to the pipe has failed,
// we don’t care.
go c.compressGoroutine(pipeWriter, destStream, *uploadCompressionFormat) // Closes pipeWriter
go c.compressGoroutine(pipeWriter, destStream, compressionMetadata, *uploadCompressionFormat) // Closes pipeWriter
destStream = pipeReader
inputInfo.Digest = ""
inputInfo.Size = -1
@ -1473,7 +1553,7 @@ func (c *copier) copyBlobFromStream(ctx context.Context, srcStream io.Reader, sr
pipeReader, pipeWriter := io.Pipe()
defer pipeReader.Close()
go c.compressGoroutine(pipeWriter, s, *uploadCompressionFormat) // Closes pipeWriter
go c.compressGoroutine(pipeWriter, s, compressionMetadata, *uploadCompressionFormat) // Closes pipeWriter
destStream = pipeReader
inputInfo.Digest = ""
@ -1533,7 +1613,7 @@ func (c *copier) copyBlobFromStream(ctx context.Context, srcStream io.Reader, sr
s, fin, err := ocicrypt.EncryptLayer(c.ociEncryptConfig, destStream, desc)
if err != nil {
return types.BlobInfo{}, errors.Wrapf(err, "Error encrypting blob %s", srcInfo.Digest)
return types.BlobInfo{}, errors.Wrapf(err, "encrypting blob %s", srcInfo.Digest)
}
destStream = s
@ -1576,7 +1656,7 @@ func (c *copier) copyBlobFromStream(ctx context.Context, srcStream io.Reader, sr
uploadedInfo, err = c.dest.PutBlob(ctx, &errorAnnotationReader{destStream}, inputInfo, c.blobInfoCache, isConfig)
}
if err != nil {
return types.BlobInfo{}, errors.Wrap(err, "Error writing blob")
return types.BlobInfo{}, errors.Wrap(err, "writing blob")
}
uploadedInfo.Annotations = srcInfo.Annotations
@ -1608,7 +1688,7 @@ func (c *copier) copyBlobFromStream(ctx context.Context, srcStream io.Reader, sr
logrus.Debugf("Consuming rest of the original blob to satisfy getOriginalLayerCopyWriter")
_, err := io.Copy(ioutil.Discard, originalLayerReader)
if err != nil {
return types.BlobInfo{}, errors.Wrapf(err, "Error reading input blob %s", srcInfo.Digest)
return types.BlobInfo{}, errors.Wrapf(err, "reading input blob %s", srcInfo.Digest)
}
}
@ -1640,23 +1720,42 @@ func (c *copier) copyBlobFromStream(ctx context.Context, srcStream io.Reader, sr
c.blobInfoCache.RecordDigestCompressorName(srcInfo.Digest, srcCompressorName)
}
}
// Copy all the metadata generated by the compressor into the annotations.
if uploadedInfo.Annotations == nil {
uploadedInfo.Annotations = map[string]string{}
}
for k, v := range compressionMetadata {
uploadedInfo.Annotations[k] = v
}
return uploadedInfo, nil
}
// doCompression reads all input from src and writes its compressed equivalent to dest.
func doCompression(dest io.Writer, src io.Reader, metadata map[string]string, compressionFormat compression.Algorithm, compressionLevel *int) error {
compressor, err := compression.CompressStreamWithMetadata(dest, metadata, compressionFormat, compressionLevel)
if err != nil {
return err
}
buf := make([]byte, compressionBufferSize)
_, err = io.CopyBuffer(compressor, src, buf) // Sets err to nil, i.e. causes dest.Close()
if err != nil {
compressor.Close()
return err
}
return compressor.Close()
}
// compressGoroutine reads all input from src and writes its compressed equivalent to dest.
func (c *copier) compressGoroutine(dest *io.PipeWriter, src io.Reader, compressionFormat compression.Algorithm) {
func (c *copier) compressGoroutine(dest *io.PipeWriter, src io.Reader, metadata map[string]string, compressionFormat compression.Algorithm) {
err := errors.New("Internal error: unexpected panic in compressGoroutine")
defer func() { // Note that this is not the same as {defer dest.CloseWithError(err)}; we need err to be evaluated lazily.
_ = dest.CloseWithError(err) // CloseWithError(nil) is equivalent to Close(), always returns nil
}()
compressor, err := compression.CompressStream(dest, compressionFormat, c.compressionLevel)
if err != nil {
return
}
defer compressor.Close()
buf := make([]byte, compressionBufferSize)
_, err = io.CopyBuffer(compressor, src, buf) // Sets err to nil, i.e. causes dest.Close()
err = doCompression(dest, src, metadata, compressionFormat, c.compressionLevel)
}

33
vendor/github.com/containers/image/v5/copy/manifest.go generated vendored
View File

@ -45,7 +45,7 @@ func (os *orderedSet) append(s string) {
func (ic *imageCopier) determineManifestConversion(ctx context.Context, destSupportedManifestMIMETypes []string, forceManifestMIMEType string, requiresOciEncryption bool) (string, []string, error) {
_, srcType, err := ic.src.Manifest(ctx)
if err != nil { // This should have been cached?!
return "", nil, errors.Wrap(err, "Error reading manifest")
return "", nil, errors.Wrap(err, "reading manifest")
}
normalizedSrcType := manifest.NormalizedMIMEType(srcType)
if srcType != normalizedSrcType {
@ -137,30 +137,29 @@ func (c *copier) determineListConversion(currentListMIMEType string, destSupport
if forcedListMIMEType != "" {
destSupportedMIMETypes = []string{forcedListMIMEType}
}
var selectedType string
var otherSupportedTypes []string
for i := range destSupportedMIMETypes {
// The second priority is the first member of the list of acceptable types that is a list,
// but keep going in case current type occurs later in the list.
if selectedType == "" && manifest.MIMETypeIsMultiImage(destSupportedMIMETypes[i]) {
selectedType = destSupportedMIMETypes[i]
}
// The first priority is the current type, if it's in the list, since that lets us avoid a
// conversion that isn't strictly necessary.
if destSupportedMIMETypes[i] == currentListMIMEType {
selectedType = destSupportedMIMETypes[i]
prioritizedTypes := newOrderedSet()
// The first priority is the current type, if it's in the list, since that lets us avoid a
// conversion that isn't strictly necessary.
for _, t := range destSupportedMIMETypes {
if t == currentListMIMEType {
prioritizedTypes.append(currentListMIMEType)
break
}
}
// Pick out the other list types that we support.
for i := range destSupportedMIMETypes {
if selectedType != destSupportedMIMETypes[i] && manifest.MIMETypeIsMultiImage(destSupportedMIMETypes[i]) {
otherSupportedTypes = append(otherSupportedTypes, destSupportedMIMETypes[i])
for _, t := range destSupportedMIMETypes {
if manifest.MIMETypeIsMultiImage(t) {
prioritizedTypes.append(t)
}
}
logrus.Debugf("Manifest list has MIME type %s, ordered candidate list [%s]", currentListMIMEType, strings.Join(destSupportedMIMETypes, ", "))
if selectedType == "" {
if len(prioritizedTypes.list) == 0 {
return "", nil, errors.Errorf("destination does not support any supported manifest list types (%v)", manifest.SupportedListMIMETypes)
}
selectedType := prioritizedTypes.list[0]
otherSupportedTypes := prioritizedTypes.list[1:]
if selectedType != currentListMIMEType {
logrus.Debugf("... will convert to %s first, and then try %v", selectedType, otherSupportedTypes)
} else {

25
vendor/github.com/containers/image/v5/copy/progress_reader.go generated vendored
View File

@ -1,9 +1,11 @@
package copy
import (
"context"
"io"
"time"
internalTypes "github.com/containers/image/v5/internal/types"
"github.com/containers/image/v5/types"
)
@ -77,3 +79,26 @@ func (r *progressReader) Read(p []byte) (int, error) {
}
return n, err
}
// imageSourceSeekableProxy wraps ImageSourceSeekable and keeps track of how many bytes
// are received.
type imageSourceSeekableProxy struct {
// source is the seekable input to read from.
source internalTypes.ImageSourceSeekable
// progress is the chan where the total number of bytes read so far are reported.
progress chan int64
}
// GetBlobAt reads from the ImageSourceSeekable and report how many bytes were received
// to the progress chan.
func (s imageSourceSeekableProxy) GetBlobAt(ctx context.Context, bInfo types.BlobInfo, chunks []internalTypes.ImageSourceChunk) (chan io.ReadCloser, chan error, error) {
rc, errs, err := s.source.GetBlobAt(ctx, bInfo, chunks)
if err == nil {
total := int64(0)
for _, c := range chunks {
total += int64(c.Length)
}
s.progress <- total
}
return rc, errs, err
}

4
vendor/github.com/containers/image/v5/copy/sign.go generated vendored
View File

@ -10,7 +10,7 @@ import (
func (c *copier) createSignature(manifest []byte, keyIdentity string) ([]byte, error) {
mech, err := signature.NewGPGSigningMechanism()
if err != nil {
return nil, errors.Wrap(err, "Error initializing GPG")
return nil, errors.Wrap(err, "initializing GPG")
}
defer mech.Close()
if err := mech.SupportsSigning(); err != nil {
@ -25,7 +25,7 @@ func (c *copier) createSignature(manifest []byte, keyIdentity string) ([]byte, e
c.Printf("Signing manifest\n")
newSig, err := signature.SignDockerManifest(manifest, dockerReference.String(), mech, keyIdentity)
if err != nil {
return nil, errors.Wrap(err, "Error creating signature")
return nil, errors.Wrap(err, "creating signature")
}
return newSig, nil
}

11
vendor/github.com/containers/image/v5/directory/directory_dest.go generated vendored
View File

@ -34,7 +34,7 @@ func newImageDestination(ref dirReference, compress bool) (types.ImageDestinatio
// if the contents don't match throw an error
dirExists, err := pathExists(d.ref.resolvedPath)
if err != nil {
return nil, errors.Wrapf(err, "error checking for path %q", d.ref.resolvedPath)
return nil, errors.Wrapf(err, "checking for path %q", d.ref.resolvedPath)
}
if dirExists {
isEmpty, err := isDirEmpty(d.ref.resolvedPath)
@ -45,7 +45,7 @@ func newImageDestination(ref dirReference, compress bool) (types.ImageDestinatio
if !isEmpty {
versionExists, err := pathExists(d.ref.versionPath())
if err != nil {
return nil, errors.Wrapf(err, "error checking if path exists %q", d.ref.versionPath())
return nil, errors.Wrapf(err, "checking if path exists %q", d.ref.versionPath())
}
if versionExists {
contents, err := ioutil.ReadFile(d.ref.versionPath())
@ -61,7 +61,7 @@ func newImageDestination(ref dirReference, compress bool) (types.ImageDestinatio
}
// delete directory contents so that only one image is in the directory at a time
if err = removeDirContents(d.ref.resolvedPath); err != nil {
return nil, errors.Wrapf(err, "error erasing contents in %q", d.ref.resolvedPath)
return nil, errors.Wrapf(err, "erasing contents in %q", d.ref.resolvedPath)
}
logrus.Debugf("overwriting existing container image directory %q", d.ref.resolvedPath)
}
@ -74,7 +74,7 @@ func newImageDestination(ref dirReference, compress bool) (types.ImageDestinatio
// create version file
err = ioutil.WriteFile(d.ref.versionPath(), []byte(version), 0644)
if err != nil {
return nil, errors.Wrapf(err, "error creating version file %q", d.ref.versionPath())
return nil, errors.Wrapf(err, "creating version file %q", d.ref.versionPath())
}
return d, nil
}
@ -239,6 +239,9 @@ func (d *dirImageDestination) PutSignatures(ctx context.Context, signatures [][]
}
// Commit marks the process of storing the image as successful and asks for the image to be persisted.
// unparsedToplevel contains data about the top-level manifest of the source (which may be a single-arch image or a manifest list
// if PutManifest was only called for the single-arch image with instanceDigest == nil), primarily to allow lookups by the
// original manifest list digest, if desired.
// WARNING: This does not have any transactional semantics:
// - Uploaded data MAY be visible to others before Commit() is called
// - Uploaded data MAY be removed or MAY remain around if Close() is called without Commit() (i.e. rollback is allowed but not guaranteed)

3
vendor/github.com/containers/image/v5/docker/archive/dest.go generated vendored
View File

@ -67,6 +67,9 @@ func (d *archiveImageDestination) Close() error {
}
// Commit marks the process of storing the image as successful and asks for the image to be persisted.
// unparsedToplevel contains data about the top-level manifest of the source (which may be a single-arch image or a manifest list
// if PutManifest was only called for the single-arch image with instanceDigest == nil), primarily to allow lookups by the
// original manifest list digest, if desired.
// WARNING: This does not have any transactional semantics:
// - Uploaded data MAY be visible to others before Commit() is called
// - Uploaded data MAY be removed or MAY remain around if Close() is called without Commit() (i.e. rollback is allowed but not guaranteed)

4
vendor/github.com/containers/image/v5/docker/archive/reader.go generated vendored
View File

@ -81,14 +81,14 @@ func (r *Reader) List() ([][]types.ImageReference, error) {
}
ref, err := newReference(r.path, nt, -1, r.archive, nil)
if err != nil {
return nil, errors.Wrapf(err, "Error creating a reference for tag %#v in manifest item @%d", tag, imageIndex)
return nil, errors.Wrapf(err, "creating a reference for tag %#v in manifest item @%d", tag, imageIndex)
}
refs = append(refs, ref)
}
if len(refs) == 0 {
ref, err := newReference(r.path, nil, imageIndex, r.archive, nil)
if err != nil {
return nil, errors.Wrapf(err, "Error creating a reference for manifest item @%d", imageIndex)
return nil, errors.Wrapf(err, "creating a reference for manifest item @%d", imageIndex)
}
refs = append(refs, ref)
}

4
vendor/github.com/containers/image/v5/docker/archive/writer.go generated vendored
View File

@ -60,7 +60,7 @@ func openArchiveForWriting(path string) (*os.File, error) {
// only in a different way. Either way, it’s up to the user to not have two writers to the same path.)
fh, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE, 0644)
if err != nil {
return nil, errors.Wrapf(err, "error opening file %q", path)
return nil, errors.Wrapf(err, "opening file %q", path)
}
succeeded := false
defer func() {
@ -70,7 +70,7 @@ func openArchiveForWriting(path string) (*os.File, error) {
}()
fhStat, err := fh.Stat()
if err != nil {
return nil, errors.Wrapf(err, "error statting file %q", path)
return nil, errors.Wrapf(err, "statting file %q", path)
}
if fhStat.Mode().IsRegular() && fhStat.Size() != 0 {

7
vendor/github.com/containers/image/v5/docker/daemon/daemon_dest.go generated vendored
View File

@ -42,7 +42,7 @@ func newImageDestination(ctx context.Context, sys *types.SystemContext, ref daem
c, err := newDockerClient(sys)
if err != nil {
return nil, errors.Wrap(err, "Error initializing docker engine client")
return nil, errors.Wrap(err, "initializing docker engine client")
}
reader, writer := io.Pipe()
@ -84,7 +84,7 @@ func imageLoadGoroutine(ctx context.Context, c *client.Client, reader *io.PipeRe
resp, err := c.ImageLoad(ctx, reader, true)
if err != nil {
err = errors.Wrap(err, "Error saving image to docker engine")
err = errors.Wrap(err, "saving image to docker engine")
return
}
defer resp.Body.Close()
@ -128,6 +128,9 @@ func (d *daemonImageDestination) Reference() types.ImageReference {
}
// Commit marks the process of storing the image as successful and asks for the image to be persisted.
// unparsedToplevel contains data about the top-level manifest of the source (which may be a single-arch image or a manifest list
// if PutManifest was only called for the single-arch image with instanceDigest == nil), primarily to allow lookups by the
// original manifest list digest, if desired.
// WARNING: This does not have any transactional semantics:
// - Uploaded data MAY be visible to others before Commit() is called
// - Uploaded data MAY be removed or MAY remain around if Close() is called without Commit() (i.e. rollback is allowed but not guaranteed)

4
vendor/github.com/containers/image/v5/docker/daemon/daemon_src.go generated vendored
View File

@ -25,13 +25,13 @@ type daemonImageSource struct {
func newImageSource(ctx context.Context, sys *types.SystemContext, ref daemonReference) (types.ImageSource, error) {
c, err := newDockerClient(sys)
if err != nil {
return nil, errors.Wrap(err, "Error initializing docker engine client")
return nil, errors.Wrap(err, "initializing docker engine client")
}
// Per NewReference(), ref.StringWithinTransport() is either an image ID (config digest), or a !reference.NameOnly() reference.
// Either way ImageSave should create a tarball with exactly one image.
inputStream, err := c.ImageSave(ctx, []string{ref.StringWithinTransport()})
if err != nil {
return nil, errors.Wrap(err, "Error loading image from docker engine")
return nil, errors.Wrap(err, "loading image from docker engine")
}
defer inputStream.Close()

34
vendor/github.com/containers/image/v5/docker/docker_client.go generated vendored
View File

@ -92,7 +92,7 @@ type bearerToken struct {
expirationTime time.Time
}
// dockerClient is configuration for dealing with a single Docker registry.
// dockerClient is configuration for dealing with a single container registry.
type dockerClient struct {
// The following members are set by newDockerClient and do not change afterwards.
sys *types.SystemContext
@ -213,10 +213,9 @@ func dockerCertDir(sys *types.SystemContext, hostPort string) (string, error) {
// “write” specifies whether the client will be used for "write" access (in particular passed to lookaside.go:toplevelFromSection)
// signatureBase is always set in the return value
func newDockerClientFromRef(sys *types.SystemContext, ref dockerReference, write bool, actions string) (*dockerClient, error) {
registry := reference.Domain(ref.ref)
auth, err := config.GetCredentials(sys, registry)
auth, err := config.GetCredentialsForRef(sys, ref.ref)
if err != nil {
return nil, errors.Wrapf(err, "error getting username and password")
return nil, errors.Wrapf(err, "getting username and password")
}
sigBase, err := SignatureStorageBaseURL(sys, ref, write)
@ -224,6 +223,7 @@ func newDockerClientFromRef(sys *types.SystemContext, ref dockerReference, write
return nil, err
}
registry := reference.Domain(ref.ref)
client, err := newDockerClient(sys, registry, ref.ref.Name())
if err != nil {
return nil, err
@ -269,7 +269,7 @@ func newDockerClient(sys *types.SystemContext, registry, reference string) (*doc
skipVerify := false
reg, err := sysregistriesv2.FindRegistry(sys, reference)
if err != nil {
return nil, errors.Wrapf(err, "error loading registries")
return nil, errors.Wrapf(err, "loading registries")
}
if reg != nil {
if reg.Blocked {
@ -297,7 +297,7 @@ func newDockerClient(sys *types.SystemContext, registry, reference string) (*doc
func CheckAuth(ctx context.Context, sys *types.SystemContext, username, password, registry string) error {
client, err := newDockerClient(sys, registry, registry)
if err != nil {
return errors.Wrapf(err, "error creating new docker client")
return errors.Wrapf(err, "creating new docker client")
}
client.auth = types.DockerAuthConfig{
Username: username,
@ -343,9 +343,10 @@ func SearchRegistry(ctx context.Context, sys *types.SystemContext, registry, ima
v1Res := &V1Results{}
// Get credentials from authfile for the underlying hostname
auth, err := config.GetCredentials(sys, registry)
// lint:ignore SA1019 We can't use GetCredentialsForRef because we want to search the whole registry.
auth, err := config.GetCredentials(sys, registry) // nolint:staticcheck // https://github.com/golangci/golangci-lint/issues/741
if err != nil {
return nil, errors.Wrapf(err, "error getting username and password")
return nil, errors.Wrapf(err, "getting username and password")
}
// The /v2/_catalog endpoint has been disabled for docker.io therefore
@ -359,7 +360,7 @@ func SearchRegistry(ctx context.Context, sys *types.SystemContext, registry, ima
client, err := newDockerClient(sys, hostname, registry)
if err != nil {
return nil, errors.Wrapf(err, "error creating new docker client")
return nil, errors.Wrapf(err, "creating new docker client")
}
client.auth = auth
if sys != nil {
@ -422,7 +423,14 @@ func SearchRegistry(ctx context.Context, sys *types.SystemContext, registry, ima
res := SearchResult{
Name: repo,
}
searchRes = append(searchRes, res)
// bugzilla.redhat.com/show_bug.cgi?id=1976283
// If we have a full match, make sure it's listed as the first result.
// (Note there might be a full match we never see if we reach the result limit first.)
if repo == image {
searchRes = append([]SearchResult{res}, searchRes...)
} else {
searchRes = append(searchRes, res)
}
}
}
@ -751,7 +759,7 @@ func (c *dockerClient) detectPropertiesHelper(ctx context.Context) error {
err = ping("http")
}
if err != nil {
err = errors.Wrapf(err, "error pinging docker registry %s", c.registry)
err = errors.Wrapf(err, "pinging container registry %s", c.registry)
if c.sys != nil && c.sys.DockerDisableV1Ping {
return err
}
@ -799,7 +807,7 @@ func (c *dockerClient) getExtensionsSignatures(ctx context.Context, ref dockerRe
defer res.Body.Close()
if res.StatusCode != http.StatusOK {
return nil, errors.Wrapf(clientLib.HandleErrorResponse(res), "Error downloading signatures for %s in %s", manifestDigest, ref.ref.Name())
return nil, errors.Wrapf(clientLib.HandleErrorResponse(res), "downloading signatures for %s in %s", manifestDigest, ref.ref.Name())
}
body, err := iolimits.ReadAtMost(res.Body, iolimits.MaxSignatureListBodySize)
@ -809,7 +817,7 @@ func (c *dockerClient) getExtensionsSignatures(ctx context.Context, ref dockerRe
var parsedBody extensionSignatureList
if err := json.Unmarshal(body, &parsedBody); err != nil {
return nil, errors.Wrapf(err, "Error decoding signature list")
return nil, errors.Wrapf(err, "decoding signature list")
}
return &parsedBody, nil
}

4
vendor/github.com/containers/image/v5/docker/docker_image.go generated vendored
View File

@ -73,7 +73,7 @@ func GetRepositoryTags(ctx context.Context, sys *types.SystemContext, ref types.
return nil, err
}
defer res.Body.Close()
if err := httpResponseToError(res, "Error fetching tags list"); err != nil {
if err := httpResponseToError(res, "fetching tags list"); err != nil {
return nil, err
}
@ -141,7 +141,7 @@ func GetDigest(ctx context.Context, sys *types.SystemContext, ref types.ImageRef
defer res.Body.Close()
if res.StatusCode != http.StatusOK {
return "", errors.Wrapf(registryHTTPResponseToError(res), "Error reading digest %s in %s", tagOrDigest, dr.ref.Name())
return "", errors.Wrapf(registryHTTPResponseToError(res), "reading digest %s in %s", tagOrDigest, dr.ref.Name())
}
dig, err := digest.Parse(res.Header.Get("Docker-Content-Digest"))

27
vendor/github.com/containers/image/v5/docker/docker_image_dest.go generated vendored
View File

@ -154,11 +154,11 @@ func (d *dockerImageDestination) PutBlob(ctx context.Context, stream io.Reader,
defer res.Body.Close()
if res.StatusCode != http.StatusAccepted {
logrus.Debugf("Error initiating layer upload, response %#v", *res)
return types.BlobInfo{}, errors.Wrapf(registryHTTPResponseToError(res), "Error initiating layer upload to %s in %s", uploadPath, d.c.registry)
return types.BlobInfo{}, errors.Wrapf(registryHTTPResponseToError(res), "initiating layer upload to %s in %s", uploadPath, d.c.registry)
}
uploadLocation, err := res.Location()
if err != nil {
return types.BlobInfo{}, errors.Wrap(err, "Error determining upload URL")
return types.BlobInfo{}, errors.Wrap(err, "determining upload URL")
}
digester := digest.Canonical.Digester()
@ -175,11 +175,11 @@ func (d *dockerImageDestination) PutBlob(ctx context.Context, stream io.Reader,
}
defer res.Body.Close()
if !successStatus(res.StatusCode) {
return nil, errors.Wrapf(registryHTTPResponseToError(res), "Error uploading layer chunked")
return nil, errors.Wrapf(registryHTTPResponseToError(res), "uploading layer chunked")
}
uploadLocation, err := res.Location()
if err != nil {
return nil, errors.Wrap(err, "Error determining upload URL")
return nil, errors.Wrap(err, "determining upload URL")
}
return uploadLocation, nil
}()
@ -201,7 +201,7 @@ func (d *dockerImageDestination) PutBlob(ctx context.Context, stream io.Reader,
defer res.Body.Close()
if res.StatusCode != http.StatusCreated {
logrus.Debugf("Error uploading layer, response %#v", *res)
return types.BlobInfo{}, errors.Wrapf(registryHTTPResponseToError(res), "Error uploading layer to %s", uploadLocation)
return types.BlobInfo{}, errors.Wrapf(registryHTTPResponseToError(res), "uploading layer to %s", uploadLocation)
}
logrus.Debugf("Upload of layer %s complete", computedDigest)
@ -226,7 +226,7 @@ func (d *dockerImageDestination) blobExists(ctx context.Context, repo reference.
return true, getBlobSize(res), nil
case http.StatusUnauthorized:
logrus.Debugf("... not authorized")
return false, -1, errors.Wrapf(registryHTTPResponseToError(res), "Error checking whether a blob %s exists in %s", digest, repo.Name())
return false, -1, errors.Wrapf(registryHTTPResponseToError(res), "checking whether a blob %s exists in %s", digest, repo.Name())
case http.StatusNotFound:
logrus.Debugf("... not present")
return false, -1, nil
@ -261,7 +261,7 @@ func (d *dockerImageDestination) mountBlob(ctx context.Context, srcRepo referenc
// NOTE: This does not really work in docker/distribution servers, which incorrectly require the "delete" action in the token's scope, and is thus entirely untested.
uploadLocation, err := res.Location()
if err != nil {
return errors.Wrap(err, "Error determining upload URL after a mount attempt")
return errors.Wrap(err, "determining upload URL after a mount attempt")
}
logrus.Debugf("... started an upload instead of mounting, trying to cancel at %s", uploadLocation.String())
res2, err := d.c.makeRequestToResolvedURL(ctx, "DELETE", uploadLocation.String(), nil, nil, -1, v2Auth, extraScope)
@ -277,7 +277,7 @@ func (d *dockerImageDestination) mountBlob(ctx context.Context, srcRepo referenc
return fmt.Errorf("Mounting %s from %s to %s started an upload instead", srcDigest, srcRepo.Name(), d.ref.ref.Name())
default:
logrus.Debugf("Error mounting, response %#v", *res)
return errors.Wrapf(registryHTTPResponseToError(res), "Error mounting %s from %s to %s", srcDigest, srcRepo.Name(), d.ref.ref.Name())
return errors.Wrapf(registryHTTPResponseToError(res), "mounting %s from %s to %s", srcDigest, srcRepo.Name(), d.ref.ref.Name())
}
}
@ -392,7 +392,7 @@ func (d *dockerImageDestination) PutManifest(ctx context.Context, m []byte, inst
// Double-check that the manifest we've been given matches the digest we've been given.
matches, err := manifest.MatchesDigest(m, *instanceDigest)
if err != nil {
return errors.Wrapf(err, "error digesting manifest in PutManifest")
return errors.Wrapf(err, "digesting manifest in PutManifest")
}
if !matches {
manifestDigest, merr := manifest.Digest(m)
@ -430,7 +430,7 @@ func (d *dockerImageDestination) PutManifest(ctx context.Context, m []byte, inst
}
defer res.Body.Close()
if !successStatus(res.StatusCode) {
err = errors.Wrapf(registryHTTPResponseToError(res), "Error uploading manifest %s to %s", refTail, d.ref.ref.Name())
err = errors.Wrapf(registryHTTPResponseToError(res), "uploading manifest %s to %s", refTail, d.ref.ref.Name())
if isManifestInvalidError(errors.Cause(err)) {
err = types.ManifestTypeRejectedError{Err: err}
}
@ -621,7 +621,7 @@ sigExists:
randBytes := make([]byte, 16)
n, err := rand.Read(randBytes)
if err != nil || n != 16 {
return errors.Wrapf(err, "Error generating random signature len %d", n)
return errors.Wrapf(err, "generating random signature len %d", n)
}
signatureName = fmt.Sprintf("%s@%032x", manifestDigest.String(), randBytes)
if _, ok := existingSigNames[signatureName]; !ok {
@ -651,7 +651,7 @@ sigExists:
logrus.Debugf("Error body %s", string(body))
}
logrus.Debugf("Error uploading signature, status %d, %#v", res.StatusCode, res)
return errors.Wrapf(registryHTTPResponseToError(res), "Error uploading signature to %s in %s", path, d.c.registry)
return errors.Wrapf(registryHTTPResponseToError(res), "uploading signature to %s in %s", path, d.c.registry)
}
}
@ -659,6 +659,9 @@ sigExists:
}
// Commit marks the process of storing the image as successful and asks for the image to be persisted.
// unparsedToplevel contains data about the top-level manifest of the source (which may be a single-arch image or a manifest list
// if PutManifest was only called for the single-arch image with instanceDigest == nil), primarily to allow lookups by the
// original manifest list digest, if desired.
// WARNING: This does not have any transactional semantics:
// - Uploaded data MAY be visible to others before Commit() is called
// - Uploaded data MAY be removed or MAY remain around if Close() is called without Commit() (i.e. rollback is allowed but not guaranteed)

200
vendor/github.com/containers/image/v5/docker/docker_image_src.go generated vendored
View File

@ -6,14 +6,17 @@ import (
"io"
"io/ioutil"
"mime"
"mime/multipart"
"net/http"
"net/url"
"os"
"strconv"
"strings"
"sync"
"github.com/containers/image/v5/docker/reference"
"github.com/containers/image/v5/internal/iolimits"
internalTypes "github.com/containers/image/v5/internal/types"
"github.com/containers/image/v5/manifest"
"github.com/containers/image/v5/pkg/sysregistriesv2"
"github.com/containers/image/v5/types"
@ -36,7 +39,7 @@ type dockerImageSource struct {
func newImageSource(ctx context.Context, sys *types.SystemContext, ref dockerReference) (*dockerImageSource, error) {
registry, err := sysregistriesv2.FindRegistry(sys, ref.ref.Name())
if err != nil {
return nil, errors.Wrapf(err, "error loading registries configuration")
return nil, errors.Wrapf(err, "loading registries configuration")
}
if registry == nil {
// No configuration was found for the provided reference, so use the
@ -69,7 +72,6 @@ func newImageSource(ctx context.Context, sys *types.SystemContext, ref dockerRef
} else {
logrus.Debugf("Trying to access %q", pullSource.Reference)
}
logrus.Debugf("Trying to access %q", pullSource.Reference)
s, err := newImageSourceAttempt(ctx, sys, ref, pullSource)
if err == nil {
return s, nil
@ -197,7 +199,7 @@ func (s *dockerImageSource) fetchManifest(ctx context.Context, tagOrDigest strin
logrus.Debugf("Content-Type from manifest GET is %q", res.Header.Get("Content-Type"))
defer res.Body.Close()
if res.StatusCode != http.StatusOK {
return nil, "", errors.Wrapf(registryHTTPResponseToError(res), "Error reading manifest %s in %s", tagOrDigest, s.physicalRef.ref.Name())
return nil, "", errors.Wrapf(registryHTTPResponseToError(res), "reading manifest %s in %s", tagOrDigest, s.physicalRef.ref.Name())
}
manblob, err := iolimits.ReadAtMost(res.Body, iolimits.MaxManifestBodySize)
@ -276,6 +278,82 @@ func (s *dockerImageSource) HasThreadSafeGetBlob() bool {
return true
}
// GetBlobAt returns a stream for the specified blob.
func (s *dockerImageSource) GetBlobAt(ctx context.Context, info types.BlobInfo, chunks []internalTypes.ImageSourceChunk) (chan io.ReadCloser, chan error, error) {
headers := make(map[string][]string)
var rangeVals []string
for _, c := range chunks {
rangeVals = append(rangeVals, fmt.Sprintf("%d-%d", c.Offset, c.Offset+c.Length-1))
}
headers["Range"] = []string{fmt.Sprintf("bytes=%s", strings.Join(rangeVals, ","))}
if len(info.URLs) != 0 {
return nil, nil, fmt.Errorf("external URLs not supported with GetBlobAt")
}
path := fmt.Sprintf(blobsPath, reference.Path(s.physicalRef.ref), info.Digest.String())
logrus.Debugf("Downloading %s", path)
res, err := s.c.makeRequest(ctx, "GET", path, headers, nil, v2Auth, nil)
if err != nil {
return nil, nil, err
}
if err := httpResponseToError(res, "Error fetching partial blob"); err != nil {
if res.Body != nil {
res.Body.Close()
}
return nil, nil, err
}
if res.StatusCode != http.StatusPartialContent {
res.Body.Close()
return nil, nil, errors.Errorf("invalid status code returned when fetching blob %d (%s)", res.StatusCode, http.StatusText(res.StatusCode))
}
mediaType, params, err := mime.ParseMediaType(res.Header.Get("Content-Type"))
if err != nil {
return nil, nil, err
}
streams := make(chan io.ReadCloser)
errs := make(chan error)
go func() {
defer close(streams)
defer close(errs)
if !strings.HasPrefix(mediaType, "multipart/") {
streams <- res.Body
return
}
boundary, found := params["boundary"]
if !found {
errs <- errors.Errorf("could not find boundary")
return
}
buffered := makeBufferedNetworkReader(res.Body, 64, 16384)
defer buffered.Close()
mr := multipart.NewReader(buffered, boundary)
for {
p, err := mr.NextPart()
if err != nil {
if err != io.EOF {
errs <- err
}
return
}
s := signalCloseReader{
Closed: make(chan interface{}),
Stream: p,
}
streams <- s
// NextPart() cannot be called while the current part
// is being read, so wait until it is closed
<-s.Closed
}
}()
return streams, errs, nil
}
// GetBlob returns a stream for the specified blob, and the blob’s size (or -1 if unknown).
// The Digest field in BlobInfo is guaranteed to be provided, Size may be -1 and MediaType may be optionally provided.
// May update BlobInfoCache, preferably after it knows for certain that a blob truly exists at a specific location.
@ -499,3 +577,119 @@ func deleteImage(ctx context.Context, sys *types.SystemContext, ref dockerRefere
return nil
}
type bufferedNetworkReaderBuffer struct {
data []byte
len int
consumed int
err error
}
type bufferedNetworkReader struct {
stream io.Reader
emptyBuffer chan *bufferedNetworkReaderBuffer
readyBuffer chan *bufferedNetworkReaderBuffer
terminate chan bool
current *bufferedNetworkReaderBuffer
mutex sync.Mutex
gotEOF bool
}
// handleBufferedNetworkReader runs in a goroutine
func handleBufferedNetworkReader(br *bufferedNetworkReader) {
defer close(br.readyBuffer)
for {
select {
case b := <-br.emptyBuffer:
b.len, b.err = br.stream.Read(b.data)
br.readyBuffer <- b
if b.err != nil {
return
}
case <-br.terminate:
return
}
}
}
func (n *bufferedNetworkReader) Close() {
close(n.terminate)
close(n.emptyBuffer)
}
func (n *bufferedNetworkReader) read(p []byte) (int, error) {
if n.current != nil {
copied := copy(p, n.current.data[n.current.consumed:n.current.len])
n.current.consumed += copied
if n.current.consumed == n.current.len {
n.emptyBuffer <- n.current
n.current = nil
}
if copied > 0 {
return copied, nil
}
}
if n.gotEOF {
return 0, io.EOF
}
var b *bufferedNetworkReaderBuffer
select {
case b = <-n.readyBuffer:
if b.err != nil {
if b.err != io.EOF {
return b.len, b.err
}
n.gotEOF = true
}
b.consumed = 0
n.current = b
return n.read(p)
case <-n.terminate:
return 0, io.EOF
}
}
func (n *bufferedNetworkReader) Read(p []byte) (int, error) {
n.mutex.Lock()
defer n.mutex.Unlock()
return n.read(p)
}
func makeBufferedNetworkReader(stream io.Reader, nBuffers, bufferSize uint) *bufferedNetworkReader {
br := bufferedNetworkReader{
stream: stream,
emptyBuffer: make(chan *bufferedNetworkReaderBuffer, nBuffers),
readyBuffer: make(chan *bufferedNetworkReaderBuffer, nBuffers),
terminate: make(chan bool),
}
go func() {
handleBufferedNetworkReader(&br)
}()
for i := uint(0); i < nBuffers; i++ {
b := bufferedNetworkReaderBuffer{
data: make([]byte, bufferSize),
}
br.emptyBuffer <- &b
}
return &br
}
type signalCloseReader struct {
Closed chan interface{}
Stream io.ReadCloser
}
func (s signalCloseReader) Read(p []byte) (int, error) {
return s.Stream.Read(p)
}
func (s signalCloseReader) Close() error {
defer close(s.Closed)
return s.Stream.Close()
}

2
vendor/github.com/containers/image/v5/docker/docker_transport.go generated vendored
View File

@ -16,7 +16,7 @@ func init() {
transports.Register(Transport)
}
// Transport is an ImageTransport for Docker registry-hosted images.
// Transport is an ImageTransport for container registry-hosted images.
var Transport = dockerTransport{}
type dockerTransport struct{}

7
vendor/github.com/containers/image/v5/docker/errors.go generated vendored
View File

@ -5,6 +5,7 @@ import (
"fmt"
"net/http"
internalTypes "github.com/containers/image/v5/internal/types"
"github.com/docker/distribution/registry/client"
perrors "github.com/pkg/errors"
)
@ -12,7 +13,7 @@ import (
var (
// ErrV1NotSupported is returned when we're trying to talk to a
// docker V1 registry.
ErrV1NotSupported = errors.New("can't talk to a V1 docker registry")
ErrV1NotSupported = errors.New("can't talk to a V1 container registry")
// ErrTooManyRequests is returned when the status code returned is 429
ErrTooManyRequests = errors.New("too many requests to registry")
)
@ -32,11 +33,15 @@ func httpResponseToError(res *http.Response, context string) error {
switch res.StatusCode {
case http.StatusOK:
return nil
case http.StatusPartialContent:
return nil
case http.StatusTooManyRequests:
return ErrTooManyRequests
case http.StatusUnauthorized:
err := client.HandleErrorResponse(res)
return ErrUnauthorizedForCredentials{Err: err}
case http.StatusBadRequest:
return internalTypes.BadPartialRequestError{Status: res.Status}
default:
if context != "" {
context = context + ": "

6
vendor/github.com/containers/image/v5/docker/internal/tarfile/dest.go generated vendored
View File

@ -140,11 +140,11 @@ func (d *Destination) PutBlob(ctx context.Context, stream io.Reader, inputInfo t
if isConfig {
buf, err := iolimits.ReadAtMost(stream, iolimits.MaxConfigBodySize)
if err != nil {
return types.BlobInfo{}, errors.Wrap(err, "Error reading Config file stream")
return types.BlobInfo{}, errors.Wrap(err, "reading Config file stream")
}
d.config = buf
if err := d.archive.sendFileLocked(d.archive.configPath(inputInfo.Digest), inputInfo.Size, bytes.NewReader(buf)); err != nil {
return types.BlobInfo{}, errors.Wrap(err, "Error writing Config file")
return types.BlobInfo{}, errors.Wrap(err, "writing Config file")
}
} else {
if err := d.archive.sendFileLocked(d.archive.physicalLayerPath(inputInfo.Digest), inputInfo.Size, stream); err != nil {
@ -187,7 +187,7 @@ func (d *Destination) PutManifest(ctx context.Context, m []byte, instanceDigest
// so the caller trying a different manifest kind would be pointless.
var man manifest.Schema2
if err := json.Unmarshal(m, &man); err != nil {
return errors.Wrap(err, "Error parsing manifest")
return errors.Wrap(err, "parsing manifest")
}
if man.SchemaVersion != 2 || man.MediaType != manifest.DockerV2Schema2MediaType {
return errors.Errorf("Unsupported manifest type, need a Docker schema 2 manifest")

14
vendor/github.com/containers/image/v5/docker/internal/tarfile/reader.go generated vendored
View File

@ -30,7 +30,7 @@ type Reader struct {
func NewReaderFromFile(sys *types.SystemContext, path string) (*Reader, error) {
file, err := os.Open(path)
if err != nil {
return nil, errors.Wrapf(err, "error opening file %q", path)
return nil, errors.Wrapf(err, "opening file %q", path)
}
defer file.Close()
@ -38,7 +38,7 @@ func NewReaderFromFile(sys *types.SystemContext, path string) (*Reader, error) {
// as a source. Otherwise we pass the stream to NewReaderFromStream.
stream, isCompressed, err := compression.AutoDecompress(file)
if err != nil {
return nil, errors.Wrapf(err, "Error detecting compression for file %q", path)
return nil, errors.Wrapf(err, "detecting compression for file %q", path)
}
defer stream.Close()
if !isCompressed {
@ -55,7 +55,7 @@ func NewReaderFromStream(sys *types.SystemContext, inputStream io.Reader) (*Read
// Save inputStream to a temporary file
tarCopyFile, err := ioutil.TempFile(tmpdir.TemporaryDirectoryForBigFiles(sys), "docker-tar")
if err != nil {
return nil, errors.Wrap(err, "error creating temporary file")
return nil, errors.Wrap(err, "creating temporary file")
}
defer tarCopyFile.Close()
@ -71,7 +71,7 @@ func NewReaderFromStream(sys *types.SystemContext, inputStream io.Reader) (*Read
// giving users really confusing "invalid tar header" errors).
uncompressedStream, _, err := compression.AutoDecompress(inputStream)
if err != nil {
return nil, errors.Wrap(err, "Error auto-decompressing input")
return nil, errors.Wrap(err, "auto-decompressing input")
}
defer uncompressedStream.Close()
@ -80,7 +80,7 @@ func NewReaderFromStream(sys *types.SystemContext, inputStream io.Reader) (*Read
// TODO: This can take quite some time, and should ideally be cancellable
// using a context.Context.
if _, err := io.Copy(tarCopyFile, uncompressedStream); err != nil {
return nil, errors.Wrapf(err, "error copying contents to temporary file %q", tarCopyFile.Name())
return nil, errors.Wrapf(err, "copying contents to temporary file %q", tarCopyFile.Name())
}
succeeded = true
@ -113,7 +113,7 @@ func newReader(path string, removeOnClose bool) (*Reader, error) {
return nil, err
}
if err := json.Unmarshal(bytes, &r.Manifest); err != nil {
return nil, errors.Wrap(err, "Error decoding tar manifest.json")
return nil, errors.Wrap(err, "decoding tar manifest.json")
}
succeeded = true
@ -258,7 +258,7 @@ func findTarComponent(inputFile io.Reader, componentPath string) (*tar.Reader, *
func (r *Reader) readTarComponent(path string, limit int) ([]byte, error) {
file, err := r.openTarComponent(path)
if err != nil {
return nil, errors.Wrapf(err, "Error loading tar component %s", path)
return nil, errors.Wrapf(err, "loading tar component %s", path)
}
defer file.Close()
bytes, err := iolimits.ReadAtMost(file, limit)

8
vendor/github.com/containers/image/v5/docker/internal/tarfile/src.go generated vendored
View File

@ -80,7 +80,7 @@ func (s *Source) ensureCachedDataIsPresentPrivate() error {
}
var parsedConfig manifest.Schema2Image // There's a lot of info there, but we only really care about layer DiffIDs.
if err := json.Unmarshal(configBytes, &parsedConfig); err != nil {
return errors.Wrapf(err, "Error decoding tar config %s", tarManifest.Config)
return errors.Wrapf(err, "decoding tar config %s", tarManifest.Config)
}
if parsedConfig.RootFS == nil {
return errors.Errorf("Invalid image config (rootFS is not set): %s", tarManifest.Config)
@ -164,7 +164,7 @@ func (s *Source) prepareLayerData(tarManifest *ManifestItem, parsedConfig *manif
// the slower method of checking if it's compressed.
uncompressedStream, isCompressed, err := compression.AutoDecompress(t)
if err != nil {
return nil, errors.Wrapf(err, "Error auto-decompressing %s to determine its size", layerPath)
return nil, errors.Wrapf(err, "auto-decompressing %s to determine its size", layerPath)
}
defer uncompressedStream.Close()
@ -172,7 +172,7 @@ func (s *Source) prepareLayerData(tarManifest *ManifestItem, parsedConfig *manif
if isCompressed {
uncompressedSize, err = io.Copy(ioutil.Discard, uncompressedStream)
if err != nil {
return nil, errors.Wrapf(err, "Error reading %s to find its size", layerPath)
return nil, errors.Wrapf(err, "reading %s to find its size", layerPath)
}
}
li.size = uncompressedSize
@ -292,7 +292,7 @@ func (s *Source) GetBlob(ctx context.Context, info types.BlobInfo, cache types.B
uncompressedStream, _, err := compression.AutoDecompress(underlyingStream)
if err != nil {
return nil, 0, errors.Wrapf(err, "Error auto-decompressing blob %s", info.Digest)
return nil, 0, errors.Wrapf(err, "auto-decompressing blob %s", info.Digest)
}
newStream := uncompressedReadCloser{

16
vendor/github.com/containers/image/v5/docker/internal/tarfile/writer.go generated vendored
View File

@ -94,16 +94,16 @@ func (w *Writer) ensureSingleLegacyLayerLocked(layerID string, layerDigest diges
// See also the comment in physicalLayerPath.
physicalLayerPath := w.physicalLayerPath(layerDigest)
if err := w.sendSymlinkLocked(filepath.Join(layerID, legacyLayerFileName), filepath.Join("..", physicalLayerPath)); err != nil {
return errors.Wrap(err, "Error creating layer symbolic link")
return errors.Wrap(err, "creating layer symbolic link")
}
b := []byte("1.0")
if err := w.sendBytesLocked(filepath.Join(layerID, legacyVersionFileName), b); err != nil {
return errors.Wrap(err, "Error writing VERSION file")
return errors.Wrap(err, "writing VERSION file")
}
if err := w.sendBytesLocked(filepath.Join(layerID, legacyConfigFileName), configBytes); err != nil {
return errors.Wrap(err, "Error writing config json file")
return errors.Wrap(err, "writing config json file")
}
w.legacyLayers[layerID] = struct{}{}
@ -128,7 +128,7 @@ func (w *Writer) writeLegacyMetadataLocked(layerDescriptors []manifest.Schema2De
var config map[string]*json.RawMessage
err := json.Unmarshal(configBytes, &config)
if err != nil {
return errors.Wrap(err, "Error unmarshaling config")
return errors.Wrap(err, "unmarshaling config")
}
for _, attr := range [7]string{"architecture", "config", "container", "container_config", "created", "docker_version", "os"} {
layerConfig[attr] = config[attr]
@ -152,7 +152,7 @@ func (w *Writer) writeLegacyMetadataLocked(layerDescriptors []manifest.Schema2De
layerConfig["layer_id"] = chainID
b, err := json.Marshal(layerConfig) // Note that layerConfig["id"] is not set yet at this point.
if err != nil {
return errors.Wrap(err, "Error marshaling layer config")
return errors.Wrap(err, "marshaling layer config")
}
delete(layerConfig, "layer_id")
layerID := digest.Canonical.FromBytes(b).Hex()
@ -160,7 +160,7 @@ func (w *Writer) writeLegacyMetadataLocked(layerDescriptors []manifest.Schema2De
configBytes, err := json.Marshal(layerConfig)
if err != nil {
return errors.Wrap(err, "Error marshaling layer config")
return errors.Wrap(err, "marshaling layer config")
}
if err := w.ensureSingleLegacyLayerLocked(layerID, l.Digest, configBytes); err != nil {
@ -280,10 +280,10 @@ func (w *Writer) Close() error {
b, err = json.Marshal(w.repositories)
if err != nil {
return errors.Wrap(err, "Error marshaling repositories")
return errors.Wrap(err, "marshaling repositories")
}
if err := w.sendBytesLocked(legacyRepositoriesFileName, b); err != nil {
return errors.Wrap(err, "Error writing config json file")
return errors.Wrap(err, "writing config json file")
}
if err := w.tar.Close(); err != nil {

2
vendor/github.com/containers/image/v5/docker/lookaside.go generated vendored
View File

@ -154,7 +154,7 @@ func loadAndMergeConfig(dirPath string) (*registryConfiguration, error) {
var config registryConfiguration
err = yaml.Unmarshal(configBytes, &config)
if err != nil {
return nil, errors.Wrapf(err, "Error parsing %s", configPath)
return nil, errors.Wrapf(err, "parsing %s", configPath)
}
if config.DefaultDocker != nil {

8
vendor/github.com/containers/image/v5/image/docker_list.go generated vendored
View File

@ -11,20 +11,20 @@ import (
func manifestSchema2FromManifestList(ctx context.Context, sys *types.SystemContext, src types.ImageSource, manblob []byte) (genericManifest, error) {
list, err := manifest.Schema2ListFromManifest(manblob)
if err != nil {
return nil, errors.Wrapf(err, "Error parsing schema2 manifest list")
return nil, errors.Wrapf(err, "parsing schema2 manifest list")
}
targetManifestDigest, err := list.ChooseInstance(sys)
if err != nil {
return nil, errors.Wrapf(err, "Error choosing image instance")
return nil, errors.Wrapf(err, "choosing image instance")
}
manblob, mt, err := src.GetManifest(ctx, &targetManifestDigest)
if err != nil {
return nil, errors.Wrapf(err, "Error loading manifest for target platform")
return nil, errors.Wrapf(err, "loading manifest for target platform")
}
matches, err := manifest.MatchesDigest(manblob, targetManifestDigest)
if err != nil {
return nil, errors.Wrap(err, "Error computing manifest digest")
return nil, errors.Wrap(err, "computing manifest digest")
}
if !matches {
return nil, errors.Errorf("Image manifest does not match selected manifest digest %s", targetManifestDigest)

2
vendor/github.com/containers/image/v5/image/docker_schema2.go generated vendored
View File

@ -289,7 +289,7 @@ func (m *manifestSchema2) convertToManifestSchema1(ctx context.Context, options
// and anyway this blob is so small that it’s easier to just copy it than to worry about figuring out another location where to get it.
info, err := dest.PutBlob(ctx, bytes.NewReader(GzippedEmptyLayer), emptyLayerBlobInfo, none.NoCache, false)
if err != nil {
return nil, errors.Wrap(err, "Error uploading empty layer")
return nil, errors.Wrap(err, "uploading empty layer")
}
if info.Digest != emptyLayerBlobInfo.Digest {
return nil, errors.Errorf("Internal error: Uploaded empty layer has digest %#v instead of %s", info.Digest, emptyLayerBlobInfo.Digest)

8
vendor/github.com/containers/image/v5/image/oci_index.go generated vendored
View File

@ -11,20 +11,20 @@ import (
func manifestOCI1FromImageIndex(ctx context.Context, sys *types.SystemContext, src types.ImageSource, manblob []byte) (genericManifest, error) {
index, err := manifest.OCI1IndexFromManifest(manblob)
if err != nil {
return nil, errors.Wrapf(err, "Error parsing OCI1 index")
return nil, errors.Wrapf(err, "parsing OCI1 index")
}
targetManifestDigest, err := index.ChooseInstance(sys)
if err != nil {
return nil, errors.Wrapf(err, "Error choosing image instance")
return nil, errors.Wrapf(err, "choosing image instance")
}
manblob, mt, err := src.GetManifest(ctx, &targetManifestDigest)
if err != nil {
return nil, errors.Wrapf(err, "Error loading manifest for target platform")
return nil, errors.Wrapf(err, "loading manifest for target platform")
}
matches, err := manifest.MatchesDigest(manblob, targetManifestDigest)
if err != nil {
return nil, errors.Wrap(err, "Error computing manifest digest")
return nil, errors.Wrap(err, "computing manifest digest")
}
if !matches {
return nil, errors.Errorf("Image manifest does not match selected manifest digest %s", targetManifestDigest)

2
vendor/github.com/containers/image/v5/image/unparsed.go generated vendored
View File

@ -53,7 +53,7 @@ func (i *UnparsedImage) Manifest(ctx context.Context) ([]byte, string, error) {
if digest, haveDigest := i.expectedManifestDigest(); haveDigest {
matches, err := manifest.MatchesDigest(m, digest)
if err != nil {
return nil, "", errors.Wrap(err, "Error computing manifest digest")
return nil, "", errors.Wrap(err, "computing manifest digest")
}
if !matches {
return nil, "", errors.Errorf("Manifest does not match provided manifest digest %s", digest)

30
vendor/github.com/containers/image/v5/internal/types/types.go generated vendored
View File

@ -58,3 +58,33 @@ type TryReusingBlobOptions struct {
// The reference of the image that contains the target blob.
SrcRef reference.Named
}
// ImageSourceChunk is a portion of a blob.
// This API is experimental and can be changed without bumping the major version number.
type ImageSourceChunk struct {
Offset uint64
Length uint64
}
// ImageSourceSeekable is an image source that permits to fetch chunks of the entire blob.
// This API is experimental and can be changed without bumping the major version number.
type ImageSourceSeekable interface {
// GetBlobAt returns a stream for the specified blob.
GetBlobAt(context.Context, publicTypes.BlobInfo, []ImageSourceChunk) (chan io.ReadCloser, chan error, error)
}
// ImageDestinationPartial is a service to store a blob by requesting the missing chunks to a ImageSourceSeekable.
// This API is experimental and can be changed without bumping the major version number.
type ImageDestinationPartial interface {
// PutBlobPartial writes contents of stream and returns data representing the result.
PutBlobPartial(ctx context.Context, stream ImageSourceSeekable, srcInfo publicTypes.BlobInfo, cache publicTypes.BlobInfoCache) (publicTypes.BlobInfo, error)
}
// BadPartialRequestError is returned by ImageSourceSeekable.GetBlobAt on an invalid request.
type BadPartialRequestError struct {
Status string
}
func (e BadPartialRequestError) Error() string {
return e.Status
}

2
vendor/github.com/containers/image/v5/manifest/common.go generated vendored
View File

@ -68,7 +68,7 @@ func compressionVariantMIMEType(variantTable []compressionMIMETypeSet, mimeType
if mt == mimeType { // Found the variant
name := mtsUncompressed
if algorithm != nil {
name = algorithm.Name()
name = algorithm.InternalUnstableUndocumentedMIMEQuestionMark()
}
if res, ok := variants[name]; ok {
if res != mtsUnsupportedMIMEType {

8
vendor/github.com/containers/image/v5/manifest/docker_schema1.go generated vendored
View File

@ -109,7 +109,7 @@ func (m *Schema1) initialize() error {
m.ExtractedV1Compatibility = make([]Schema1V1Compatibility, len(m.History))
for i, h := range m.History {
if err := json.Unmarshal([]byte(h.V1Compatibility), &m.ExtractedV1Compatibility[i]); err != nil {
return errors.Wrapf(err, "Error parsing v2s1 history entry %d", i)
return errors.Wrapf(err, "parsing v2s1 history entry %d", i)
}
}
return nil
@ -242,14 +242,14 @@ func (m *Schema1) ToSchema2Config(diffIDs []digest.Digest) ([]byte, error) {
config := []byte(m.History[0].V1Compatibility)
err := json.Unmarshal(config, &s1)
if err != nil {
return nil, errors.Wrapf(err, "error decoding configuration")
return nil, errors.Wrapf(err, "decoding configuration")
}
// Images created with versions prior to 1.8.3 require us to re-encode the encoded object,
// adding some fields that aren't "omitempty".
if s1.DockerVersion != "" && versions.LessThan(s1.DockerVersion, "1.8.3") {
config, err = json.Marshal(&s1)
if err != nil {
return nil, errors.Wrapf(err, "error re-encoding compat image config %#v", s1)
return nil, errors.Wrapf(err, "re-encoding compat image config %#v", s1)
}
}
// Build the history.
@ -276,7 +276,7 @@ func (m *Schema1) ToSchema2Config(diffIDs []digest.Digest) ([]byte, error) {
raw := make(map[string]*json.RawMessage)
err = json.Unmarshal(config, &raw)
if err != nil {
return nil, errors.Wrapf(err, "error re-decoding compat image config %#v", s1)
return nil, errors.Wrapf(err, "re-decoding compat image config %#v", s1)
}
// Drop some fields.
delete(raw, "id")

2
vendor/github.com/containers/image/v5/manifest/docker_schema2.go generated vendored
View File

@ -242,7 +242,7 @@ func (m *Schema2) UpdateLayerInfos(layerInfos []types.BlobInfo) error {
}
mimeType, err := updatedMIMEType(schema2CompressionMIMETypeSets, mimeType, info)
if err != nil {
return errors.Wrapf(err, "Error preparing updated manifest, layer %q", info.Digest)
return errors.Wrapf(err, "preparing updated manifest, layer %q", info.Digest)
}
m.LayersDescriptors[i].MediaType = mimeType
m.LayersDescriptors[i].Digest = info.Digest

6
vendor/github.com/containers/image/v5/manifest/docker_schema2_list.go generated vendored
View File

@ -91,7 +91,7 @@ func (list *Schema2List) UpdateInstances(updates []ListUpdate) error {
func (list *Schema2List) ChooseInstance(ctx *types.SystemContext) (digest.Digest, error) {
wantedPlatforms, err := platform.WantedPlatforms(ctx)
if err != nil {
return "", errors.Wrapf(err, "error getting platform information %#v", ctx)
return "", errors.Wrapf(err, "getting platform information %#v", ctx)
}
for _, wantedPlatform := range wantedPlatforms {
for _, d := range list.Manifests {
@ -115,7 +115,7 @@ func (list *Schema2List) ChooseInstance(ctx *types.SystemContext) (digest.Digest
func (list *Schema2List) Serialize() ([]byte, error) {
buf, err := json.Marshal(list)
if err != nil {
return nil, errors.Wrapf(err, "error marshaling Schema2List %#v", list)
return nil, errors.Wrapf(err, "marshaling Schema2List %#v", list)
}
return buf, nil
}
@ -190,7 +190,7 @@ func Schema2ListFromManifest(manifest []byte) (*Schema2List, error) {
Manifests: []Schema2ManifestDescriptor{},
}
if err := json.Unmarshal(manifest, &list); err != nil {
return nil, errors.Wrapf(err, "error unmarshaling Schema2List %q", string(manifest))
return nil, errors.Wrapf(err, "unmarshaling Schema2List %q", string(manifest))
}
return &list, nil
}

2
vendor/github.com/containers/image/v5/manifest/manifest.go generated vendored
View File

@ -195,7 +195,7 @@ func MatchesDigest(manifest []byte, expectedDigest digest.Digest) (bool, error)
}
// AddDummyV2S1Signature adds an JWS signature with a temporary key (i.e. useless) to a v2s1 manifest.
// This is useful to make the manifest acceptable to a Docker Registry (even though nothing needs or wants the JWS signature).
// This is useful to make the manifest acceptable to a docker/distribution registry (even though nothing needs or wants the JWS signature).
func AddDummyV2S1Signature(manifest []byte) ([]byte, error) {
key, err := libtrust.GenerateECP256PrivateKey()
if err != nil {

2
vendor/github.com/containers/image/v5/manifest/oci.go generated vendored
View File

@ -127,7 +127,7 @@ func (m *OCI1) UpdateLayerInfos(layerInfos []types.BlobInfo) error {
}
mimeType, err := updatedMIMEType(oci1CompressionMIMETypeSets, mimeType, info)
if err != nil {
return errors.Wrapf(err, "Error preparing updated manifest, layer %q", info.Digest)
return errors.Wrapf(err, "preparing updated manifest, layer %q", info.Digest)
}
if info.CryptoOperation == types.Encrypt {
encMediaType, err := getEncryptedMediaType(mimeType)

6
vendor/github.com/containers/image/v5/manifest/oci_index.go generated vendored
View File

@ -75,7 +75,7 @@ func (index *OCI1Index) UpdateInstances(updates []ListUpdate) error {
func (index *OCI1Index) ChooseInstance(ctx *types.SystemContext) (digest.Digest, error) {
wantedPlatforms, err := platform.WantedPlatforms(ctx)
if err != nil {
return "", errors.Wrapf(err, "error getting platform information %#v", ctx)
return "", errors.Wrapf(err, "getting platform information %#v", ctx)
}
for _, wantedPlatform := range wantedPlatforms {
for _, d := range index.Manifests {
@ -108,7 +108,7 @@ func (index *OCI1Index) ChooseInstance(ctx *types.SystemContext) (digest.Digest,
func (index *OCI1Index) Serialize() ([]byte, error) {
buf, err := json.Marshal(index)
if err != nil {
return nil, errors.Wrapf(err, "error marshaling OCI1Index %#v", index)
return nil, errors.Wrapf(err, "marshaling OCI1Index %#v", index)
}
return buf, nil
}
@ -200,7 +200,7 @@ func OCI1IndexFromManifest(manifest []byte) (*OCI1Index, error) {
},
}
if err := json.Unmarshal(manifest, &index); err != nil {
return nil, errors.Wrapf(err, "error unmarshaling OCI1Index %q", string(manifest))
return nil, errors.Wrapf(err, "unmarshaling OCI1Index %q", string(manifest))
}
return &index, nil
}

13
vendor/github.com/containers/image/v5/oci/archive/oci_dest.go generated vendored
View File

@ -22,12 +22,12 @@ type ociArchiveImageDestination struct {
func newImageDestination(ctx context.Context, sys *types.SystemContext, ref ociArchiveReference) (types.ImageDestination, error) {
tempDirRef, err := createOCIRef(sys, ref.image)
if err != nil {
return nil, errors.Wrapf(err, "error creating oci reference")
return nil, errors.Wrapf(err, "creating oci reference")
}
unpackedDest, err := tempDirRef.ociRefExtracted.NewImageDestination(ctx, sys)
if err != nil {
if err := tempDirRef.deleteTempDir(); err != nil {
return nil, errors.Wrapf(err, "error deleting temp directory %q", tempDirRef.tempDirectory)
return nil, errors.Wrapf(err, "deleting temp directory %q", tempDirRef.tempDirectory)
}
return nil, err
}
@ -129,10 +129,13 @@ func (d *ociArchiveImageDestination) PutSignatures(ctx context.Context, signatur
}
// Commit marks the process of storing the image as successful and asks for the image to be persisted
// unparsedToplevel contains data about the top-level manifest of the source (which may be a single-arch image or a manifest list
// if PutManifest was only called for the single-arch image with instanceDigest == nil), primarily to allow lookups by the
// original manifest list digest, if desired.
// after the directory is made, it is tarred up into a file and the directory is deleted
func (d *ociArchiveImageDestination) Commit(ctx context.Context, unparsedToplevel types.UnparsedImage) error {
if err := d.unpackedDest.Commit(ctx, unparsedToplevel); err != nil {
return errors.Wrapf(err, "error storing image %q", d.ref.image)
return errors.Wrapf(err, "storing image %q", d.ref.image)
}
// path of directory to tar up
@ -147,13 +150,13 @@ func tarDirectory(src, dst string) error {
// input is a stream of bytes from the archive of the directory at path
input, err := archive.Tar(src, archive.Uncompressed)
if err != nil {
return errors.Wrapf(err, "error retrieving stream of bytes from %q", src)
return errors.Wrapf(err, "retrieving stream of bytes from %q", src)
}
// creates the tar file
outFile, err := os.Create(dst)
if err != nil {
return errors.Wrapf(err, "error creating tar file %q", dst)
return errors.Wrapf(err, "creating tar file %q", dst)
}
defer outFile.Close()

8
vendor/github.com/containers/image/v5/oci/archive/oci_src.go generated vendored
View File

@ -23,13 +23,13 @@ type ociArchiveImageSource struct {
func newImageSource(ctx context.Context, sys *types.SystemContext, ref ociArchiveReference) (types.ImageSource, error) {
tempDirRef, err := createUntarTempDir(sys, ref)
if err != nil {
return nil, errors.Wrap(err, "error creating temp directory")
return nil, errors.Wrap(err, "creating temp directory")
}
unpackedSrc, err := tempDirRef.ociRefExtracted.NewImageSource(ctx, sys)
if err != nil {
if err := tempDirRef.deleteTempDir(); err != nil {
return nil, errors.Wrapf(err, "error deleting temp directory %q", tempDirRef.tempDirectory)
return nil, errors.Wrapf(err, "deleting temp directory %q", tempDirRef.tempDirectory)
}
return nil, err
}
@ -52,7 +52,7 @@ func LoadManifestDescriptorWithContext(sys *types.SystemContext, imgRef types.Im
}
tempDirRef, err := createUntarTempDir(sys, ociArchRef)
if err != nil {
return imgspecv1.Descriptor{}, errors.Wrap(err, "error creating temp directory")
return imgspecv1.Descriptor{}, errors.Wrap(err, "creating temp directory")
}
defer func() {
err := tempDirRef.deleteTempDir()
@ -61,7 +61,7 @@ func LoadManifestDescriptorWithContext(sys *types.SystemContext, imgRef types.Im
descriptor, err := ocilayout.LoadManifestDescriptor(tempDirRef.ociRefExtracted)
if err != nil {
return imgspecv1.Descriptor{}, errors.Wrap(err, "error loading index")
return imgspecv1.Descriptor{}, errors.Wrap(err, "loading index")
}
return descriptor, nil
}

8
vendor/github.com/containers/image/v5/oci/archive/oci_transport.go generated vendored
View File

@ -163,7 +163,7 @@ func (t *tempDirOCIRef) deleteTempDir() error {
func createOCIRef(sys *types.SystemContext, image string) (tempDirOCIRef, error) {
dir, err := ioutil.TempDir(tmpdir.TemporaryDirectoryForBigFiles(sys), "oci")
if err != nil {
return tempDirOCIRef{}, errors.Wrapf(err, "error creating temp directory")
return tempDirOCIRef{}, errors.Wrapf(err, "creating temp directory")
}
ociRef, err := ocilayout.NewReference(dir, image)
if err != nil {
@ -178,7 +178,7 @@ func createOCIRef(sys *types.SystemContext, image string) (tempDirOCIRef, error)
func createUntarTempDir(sys *types.SystemContext, ref ociArchiveReference) (tempDirOCIRef, error) {
tempDirRef, err := createOCIRef(sys, ref.image)
if err != nil {
return tempDirOCIRef{}, errors.Wrap(err, "error creating oci reference")
return tempDirOCIRef{}, errors.Wrap(err, "creating oci reference")
}
src := ref.resolvedFile
dst := tempDirRef.tempDirectory
@ -190,9 +190,9 @@ func createUntarTempDir(sys *types.SystemContext, ref ociArchiveReference) (temp
defer arch.Close()
if err := archive.NewDefaultArchiver().Untar(arch, dst, &archive.TarOptions{NoLchown: true}); err != nil {
if err := tempDirRef.deleteTempDir(); err != nil {
return tempDirOCIRef{}, errors.Wrapf(err, "error deleting temp directory %q", tempDirRef.tempDirectory)
return tempDirOCIRef{}, errors.Wrapf(err, "deleting temp directory %q", tempDirRef.tempDirectory)
}
return tempDirOCIRef{}, errors.Wrapf(err, "error untarring file %q", tempDirRef.tempDirectory)
return tempDirOCIRef{}, errors.Wrapf(err, "untarring file %q", tempDirRef.tempDirectory)
}
return tempDirRef, nil
}

3
vendor/github.com/containers/image/v5/oci/layout/oci_dest.go generated vendored
View File

@ -303,6 +303,9 @@ func (d *ociImageDestination) PutSignatures(ctx context.Context, signatures [][]
}
// Commit marks the process of storing the image as successful and asks for the image to be persisted.
// unparsedToplevel contains data about the top-level manifest of the source (which may be a single-arch image or a manifest list
// if PutManifest was only called for the single-arch image with instanceDigest == nil), primarily to allow lookups by the
// original manifest list digest, if desired.
// WARNING: This does not have any transactional semantics:
// - Uploaded data MAY be visible to others before Commit() is called
// - Uploaded data MAY be removed or MAY remain around if Close() is called without Commit() (i.e. rollback is allowed but not guaranteed)

2
vendor/github.com/containers/image/v5/openshift/openshift-copies.go generated vendored
View File

@ -579,7 +579,7 @@ func (rules *clientConfigLoadingRules) Load() (*clientcmdConfig, error) {
continue
}
if err != nil {
errlist = append(errlist, errors.Wrapf(err, "Error loading config file \"%s\"", filename))
errlist = append(errlist, errors.Wrapf(err, "loading config file \"%s\"", filename))
continue
}

19
vendor/github.com/containers/image/v5/openshift/openshift.go generated vendored
View File

@ -164,7 +164,7 @@ type openshiftImageSource struct {
// Values specific to this image
sys *types.SystemContext
// State
docker types.ImageSource // The Docker Registry endpoint, or nil if not resolved yet
docker types.ImageSource // The docker/distribution API endpoint, or nil if not resolved yet
imageStreamImageName string // Resolved image identifier, or "" if not known yet
}
@ -316,7 +316,7 @@ func (s *openshiftImageSource) ensureImageIsResolved(ctx context.Context) error
type openshiftImageDestination struct {
client *openshiftClient
docker types.ImageDestination // The Docker Registry endpoint
docker types.ImageDestination // The docker/distribution API endpoint
// State
imageStreamImageName string // "" if not yet known
}
@ -435,14 +435,14 @@ func (d *openshiftImageDestination) PutManifest(ctx context.Context, m []byte, i
}
func (d *openshiftImageDestination) PutSignatures(ctx context.Context, signatures [][]byte, instanceDigest *digest.Digest) error {
var imageStreamName string
var imageStreamImageName string
if instanceDigest == nil {
if d.imageStreamImageName == "" {
return errors.Errorf("Internal error: Unknown manifest digest, can't add signatures")
}
imageStreamName = d.imageStreamImageName
imageStreamImageName = d.imageStreamImageName
} else {
imageStreamName = instanceDigest.String()
imageStreamImageName = instanceDigest.String()
}
// Because image signatures are a shared resource in Atomic Registry, the default upload
@ -452,7 +452,7 @@ func (d *openshiftImageDestination) PutSignatures(ctx context.Context, signature
return nil // No need to even read the old state.
}
image, err := d.client.getImage(ctx, imageStreamName)
image, err := d.client.getImage(ctx, imageStreamImageName)
if err != nil {
return err
}
@ -475,9 +475,9 @@ sigExists:
randBytes := make([]byte, 16)
n, err := rand.Read(randBytes)
if err != nil || n != 16 {
return errors.Wrapf(err, "Error generating random signature len %d", n)
return errors.Wrapf(err, "generating random signature len %d", n)
}
signatureName = fmt.Sprintf("%s@%032x", imageStreamName, randBytes)
signatureName = fmt.Sprintf("%s@%032x", imageStreamImageName, randBytes)
if _, ok := existingSigNames[signatureName]; !ok {
break
}
@ -506,6 +506,9 @@ sigExists:
}
// Commit marks the process of storing the image as successful and asks for the image to be persisted.
// unparsedToplevel contains data about the top-level manifest of the source (which may be a single-arch image or a manifest list
// if PutManifest was only called for the single-arch image with instanceDigest == nil), primarily to allow lookups by the
// original manifest list digest, if desired.
// WARNING: This does not have any transactional semantics:
// - Uploaded data MAY be visible to others before Commit() is called
// - Uploaded data MAY be removed or MAY remain around if Close() is called without Commit() (i.e. rollback is allowed but not guaranteed)

39
vendor/github.com/containers/image/v5/pkg/compression/compression.go generated vendored
View File

@ -9,6 +9,7 @@ import (
"github.com/containers/image/v5/pkg/compression/internal"
"github.com/containers/image/v5/pkg/compression/types"
"github.com/containers/storage/pkg/chunked"
"github.com/klauspost/pgzip"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
@ -20,19 +21,22 @@ type Algorithm = types.Algorithm
var (
// Gzip compression.
Gzip = internal.NewAlgorithm("gzip", []byte{0x1F, 0x8B, 0x08}, GzipDecompressor, gzipCompressor)
Gzip = internal.NewAlgorithm("gzip", "gzip", []byte{0x1F, 0x8B, 0x08}, GzipDecompressor, gzipCompressor)
// Bzip2 compression.
Bzip2 = internal.NewAlgorithm("bzip2", []byte{0x42, 0x5A, 0x68}, Bzip2Decompressor, bzip2Compressor)
Bzip2 = internal.NewAlgorithm("bzip2", "bzip2", []byte{0x42, 0x5A, 0x68}, Bzip2Decompressor, bzip2Compressor)
// Xz compression.
Xz = internal.NewAlgorithm("Xz", []byte{0xFD, 0x37, 0x7A, 0x58, 0x5A, 0x00}, XzDecompressor, xzCompressor)
Xz = internal.NewAlgorithm("Xz", "xz", []byte{0xFD, 0x37, 0x7A, 0x58, 0x5A, 0x00}, XzDecompressor, xzCompressor)
// Zstd compression.
Zstd = internal.NewAlgorithm("zstd", []byte{0x28, 0xb5, 0x2f, 0xfd}, ZstdDecompressor, zstdCompressor)
Zstd = internal.NewAlgorithm("zstd", "zstd", []byte{0x28, 0xb5, 0x2f, 0xfd}, ZstdDecompressor, zstdCompressor)
// Zstd:chunked compression.
ZstdChunked = internal.NewAlgorithm("zstd:chunked", "zstd", []byte{0x28, 0xb5, 0x2f, 0xfd}, ZstdDecompressor, chunked.ZstdCompressor)
compressionAlgorithms = map[string]Algorithm{
Gzip.Name(): Gzip,
Bzip2.Name(): Bzip2,
Xz.Name(): Xz,
Zstd.Name(): Zstd,
Gzip.Name(): Gzip,
Bzip2.Name(): Bzip2,
Xz.Name(): Xz,
Zstd.Name(): Zstd,
ZstdChunked.Name(): ZstdChunked,
}
)
@ -69,7 +73,7 @@ func XzDecompressor(r io.Reader) (io.ReadCloser, error) {
}
// gzipCompressor is a CompressorFunc for the gzip compression algorithm.
func gzipCompressor(r io.Writer, level *int) (io.WriteCloser, error) {
func gzipCompressor(r io.Writer, metadata map[string]string, level *int) (io.WriteCloser, error) {
if level != nil {
return pgzip.NewWriterLevel(r, *level)
}
@ -77,18 +81,25 @@ func gzipCompressor(r io.Writer, level *int) (io.WriteCloser, error) {
}
// bzip2Compressor is a CompressorFunc for the bzip2 compression algorithm.
func bzip2Compressor(r io.Writer, level *int) (io.WriteCloser, error) {
func bzip2Compressor(r io.Writer, metadata map[string]string, level *int) (io.WriteCloser, error) {
return nil, fmt.Errorf("bzip2 compression not supported")
}
// xzCompressor is a CompressorFunc for the xz compression algorithm.
func xzCompressor(r io.Writer, level *int) (io.WriteCloser, error) {
func xzCompressor(r io.Writer, metadata map[string]string, level *int) (io.WriteCloser, error) {
return xz.NewWriter(r)
}
// CompressStream returns the compressor by its name
func CompressStream(dest io.Writer, algo Algorithm, level *int) (io.WriteCloser, error) {
return internal.AlgorithmCompressor(algo)(dest, level)
m := map[string]string{}
return internal.AlgorithmCompressor(algo)(dest, m, level)
}
// CompressStreamWithMetadata returns the compressor by its name. If the compression
// generates any metadata, it is written to the provided metadata map.
func CompressStreamWithMetadata(dest io.Writer, metadata map[string]string, algo Algorithm, level *int) (io.WriteCloser, error) {
return internal.AlgorithmCompressor(algo)(dest, metadata, level)
}
// DetectCompressionFormat returns an Algorithm and DecompressorFunc if the input is recognized as a compressed format, an invalid
@ -135,13 +146,13 @@ func DetectCompression(input io.Reader) (DecompressorFunc, io.Reader, error) {
func AutoDecompress(stream io.Reader) (io.ReadCloser, bool, error) {
decompressor, stream, err := DetectCompression(stream)
if err != nil {
return nil, false, errors.Wrapf(err, "Error detecting compression")
return nil, false, errors.Wrapf(err, "detecting compression")
}
var res io.ReadCloser
if decompressor != nil {
res, err = decompressor(stream)
if err != nil {
return nil, false, errors.Wrapf(err, "Error initializing decompression")
return nil, false, errors.Wrapf(err, "initializing decompression")
}
} else {
res = ioutil.NopCloser(stream)

12
vendor/github.com/containers/image/v5/pkg/compression/internal/types.go generated vendored
View File

@ -4,7 +4,7 @@ import "io"
// CompressorFunc writes the compressed stream to the given writer using the specified compression level.
// The caller must call Close() on the stream (even if the input stream does not need closing!).
type CompressorFunc func(io.Writer, *int) (io.WriteCloser, error)
type CompressorFunc func(io.Writer, map[string]string, *int) (io.WriteCloser, error)
// DecompressorFunc returns the decompressed stream, given a compressed stream.
// The caller must call Close() on the decompressed stream (even if the compressed input stream does not need closing!).
@ -13,6 +13,7 @@ type DecompressorFunc func(io.Reader) (io.ReadCloser, error)
// Algorithm is a compression algorithm that can be used for CompressStream.
type Algorithm struct {
name string
mime string
prefix []byte
decompressor DecompressorFunc
compressor CompressorFunc
@ -21,9 +22,10 @@ type Algorithm struct {
// NewAlgorithm creates an Algorithm instance.
// This function exists so that Algorithm instances can only be created by code that
// is allowed to import this internal subpackage.
func NewAlgorithm(name string, prefix []byte, decompressor DecompressorFunc, compressor CompressorFunc) Algorithm {
func NewAlgorithm(name, mime string, prefix []byte, decompressor DecompressorFunc, compressor CompressorFunc) Algorithm {
return Algorithm{
name: name,
mime: mime,
prefix: prefix,
decompressor: decompressor,
compressor: compressor,
@ -35,6 +37,12 @@ func (c Algorithm) Name() string {
return c.name
}
// InternalUnstableUndocumentedMIMEQuestionMark ???
// DO NOT USE THIS anywhere outside of c/image until it is properly documented.
func (c Algorithm) InternalUnstableUndocumentedMIMEQuestionMark() string {
return c.mime
}
// AlgorithmCompressor returns the compressor field of algo.
// This is a function instead of a public method so that it is only callable from by code
// that is allowed to import this internal subpackage.

4
vendor/github.com/containers/image/v5/pkg/compression/zstd.go generated vendored
View File

@ -40,13 +40,13 @@ func zstdWriter(dest io.Writer) (io.WriteCloser, error) {
return zstd.NewWriter(dest)
}
func zstdWriterWithLevel(dest io.Writer, level int) (io.WriteCloser, error) {
func zstdWriterWithLevel(dest io.Writer, level int) (*zstd.Encoder, error) {
el := zstd.EncoderLevelFromZstd(level)
return zstd.NewWriter(dest, zstd.WithEncoderLevel(el))
}
// zstdCompressor is a CompressorFunc for the zstd compression algorithm.
func zstdCompressor(r io.Writer, level *int) (io.WriteCloser, error) {
func zstdCompressor(r io.Writer, metadata map[string]string, level *int) (io.WriteCloser, error) {
if level == nil {
return zstdWriter(r)
}

235
vendor/github.com/containers/image/v5/pkg/docker/config/config.go generated vendored
View File

@ -11,6 +11,7 @@ import (
"runtime"
"strings"
"github.com/containers/image/v5/docker/reference"
"github.com/containers/image/v5/pkg/sysregistriesv2"
"github.com/containers/image/v5/types"
"github.com/containers/storage/pkg/homedir"
@ -51,12 +52,19 @@ var (
ErrNotSupported = errors.New("not supported")
)
// SetCredentials stores the username and password in the credential helper or file
// and returns path to file or helper name in format (helper:%s).
// SetCredentials stores the username and password in a location
// appropriate for sys and the users’ configuration.
// A valid key can be either a registry hostname or additionally a namespace if
// the AuthenticationFileHelper is being unsed.
// Returns a human-redable description of the location that was updated.
// NOTE: The return value is only intended to be read by humans; its form is not an API,
// it may change (or new forms can be added) any time.
func SetCredentials(sys *types.SystemContext, registry, username, password string) (string, error) {
func SetCredentials(sys *types.SystemContext, key, username, password string) (string, error) {
isNamespaced, err := validateKey(key)
if err != nil {
return "", err
}
helpers, err := sysregistriesv2.CredentialHelpers(sys)
if err != nil {
return "", err
@ -71,33 +79,45 @@ func SetCredentials(sys *types.SystemContext, registry, username, password strin
// Special-case the built-in helpers for auth files.
case sysregistriesv2.AuthenticationFileHelper:
desc, err = modifyJSON(sys, func(auths *dockerConfigFile) (bool, error) {
if ch, exists := auths.CredHelpers[registry]; exists {
return false, setAuthToCredHelper(ch, registry, username, password)
if ch, exists := auths.CredHelpers[key]; exists {
if isNamespaced {
return false, unsupportedNamespaceErr(ch)
}
return false, setAuthToCredHelper(ch, key, username, password)
}
creds := base64.StdEncoding.EncodeToString([]byte(username + ":" + password))
newCreds := dockerAuthConfig{Auth: creds}
auths.AuthConfigs[registry] = newCreds
auths.AuthConfigs[key] = newCreds
return true, nil
})
// External helpers.
default:
desc = fmt.Sprintf("credential helper: %s", helper)
err = setAuthToCredHelper(helper, registry, username, password)
if isNamespaced {
err = unsupportedNamespaceErr(helper)
} else {
desc = fmt.Sprintf("credential helper: %s", helper)
err = setAuthToCredHelper(helper, key, username, password)
}
}
if err != nil {
multiErr = multierror.Append(multiErr, err)
logrus.Debugf("Error storing credentials for %s in credential helper %s: %v", registry, helper, err)
logrus.Debugf("Error storing credentials for %s in credential helper %s: %v", key, helper, err)
continue
}
logrus.Debugf("Stored credentials for %s in credential helper %s", registry, helper)
logrus.Debugf("Stored credentials for %s in credential helper %s", key, helper)
return desc, nil
}
return "", multiErr
}
func unsupportedNamespaceErr(helper string) error {
return errors.Errorf("namespaced key is not supported for credential helper %s", helper)
}
// SetAuthentication stores the username and password in the credential helper or file
func SetAuthentication(sys *types.SystemContext, registry, username, password string) error {
_, err := SetCredentials(sys, registry, username, password)
// See the documentation of SetCredentials for format of "key"
func SetAuthentication(sys *types.SystemContext, key, username, password string) error {
_, err := SetCredentials(sys, key, username, password)
return err
}
@ -125,7 +145,7 @@ func GetAllCredentials(sys *types.SystemContext) (map[string]types.DockerAuthCon
// readJSONFile returns an empty map in case the path doesn't exist.
auths, err := readJSONFile(path.path, path.legacyFormat)
if err != nil {
return nil, errors.Wrapf(err, "error reading JSON file %q", path.path)
return nil, errors.Wrapf(err, "reading JSON file %q", path.path)
}
// Credential helpers in the auth file have a
// direct mapping to a registry, so we can just
@ -215,13 +235,34 @@ func getAuthFilePaths(sys *types.SystemContext, homeDir string) []authPath {
// helpers with falling back to using either auth.json
// file or .docker/config.json, including support for OAuth2 and IdentityToken.
// If an entry is not found, an empty struct is returned.
//
// Deprecated: GetCredentialsForRef should be used in favor of this API
// because it allows different credentials for different repositories on the
// same registry.
func GetCredentials(sys *types.SystemContext, registry string) (types.DockerAuthConfig, error) {
return getCredentialsWithHomeDir(sys, registry, homedir.Get())
return getCredentialsWithHomeDir(sys, nil, registry, homedir.Get())
}
// getCredentialsWithHomeDir is an internal implementation detail of GetCredentials,
// it exists only to allow testing it with an artificial home directory.
func getCredentialsWithHomeDir(sys *types.SystemContext, registry, homeDir string) (types.DockerAuthConfig, error) {
// GetCredentialsForRef returns the registry credentials necessary for
// accessing ref on the registry ref points to,
// appropriate for sys and the users’ configuration.
// If an entry is not found, an empty struct is returned.
func GetCredentialsForRef(sys *types.SystemContext, ref reference.Named) (types.DockerAuthConfig, error) {
return getCredentialsWithHomeDir(sys, ref, reference.Domain(ref), homedir.Get())
}
// getCredentialsWithHomeDir is an internal implementation detail of
// GetCredentialsForRef and GetCredentials. It exists only to allow testing it
// with an artificial home directory.
func getCredentialsWithHomeDir(sys *types.SystemContext, ref reference.Named, registry, homeDir string) (types.DockerAuthConfig, error) {
// consistency check of the ref and registry arguments
if ref != nil && reference.Domain(ref) != registry {
return types.DockerAuthConfig{}, errors.Errorf(
"internal error: provided reference domain %q name does not match registry %q",
reference.Domain(ref), registry,
)
}
if sys != nil && sys.DockerAuthConfig != nil {
logrus.Debugf("Returning credentials for %s from DockerAuthConfig", registry)
return *sys.DockerAuthConfig, nil
@ -230,7 +271,7 @@ func getCredentialsWithHomeDir(sys *types.SystemContext, registry, homeDir strin
// Anonymous function to query credentials from auth files.
getCredentialsFromAuthFiles := func() (types.DockerAuthConfig, error) {
for _, path := range getAuthFilePaths(sys, homeDir) {
authConfig, err := findAuthentication(registry, path.path, path.legacyFormat)
authConfig, err := findAuthentication(ref, registry, path.path, path.legacyFormat)
if err != nil {
return types.DockerAuthConfig{}, err
}
@ -284,7 +325,7 @@ func getCredentialsWithHomeDir(sys *types.SystemContext, registry, homeDir strin
// .docker/config.json
//
// Deprecated: This API only has support for username and password. To get the
// support for oauth2 in docker registry authentication, we added the new
// support for oauth2 in container registry authentication, we added the new
// GetCredentials API. The new API should be used and this API is kept to
// maintain backward compatibility.
func GetAuthentication(sys *types.SystemContext, registry string) (string, string, error) {
@ -294,7 +335,7 @@ func GetAuthentication(sys *types.SystemContext, registry string) (string, strin
// getAuthenticationWithHomeDir is an internal implementation detail of GetAuthentication,
// it exists only to allow testing it with an artificial home directory.
func getAuthenticationWithHomeDir(sys *types.SystemContext, registry, homeDir string) (string, string, error) {
auth, err := getCredentialsWithHomeDir(sys, registry, homeDir)
auth, err := getCredentialsWithHomeDir(sys, nil, registry, homeDir)
if err != nil {
return "", "", err
}
@ -304,9 +345,16 @@ func getAuthenticationWithHomeDir(sys *types.SystemContext, registry, homeDir st
return auth.Username, auth.Password, nil
}
// RemoveAuthentication removes credentials for `registry` from all possible
// RemoveAuthentication removes credentials for `key` from all possible
// sources such as credential helpers and auth files.
func RemoveAuthentication(sys *types.SystemContext, registry string) error {
// A valid key can be either a registry hostname or additionally a namespace if
// the AuthenticationFileHelper is being unsed.
func RemoveAuthentication(sys *types.SystemContext, key string) error {
isNamespaced, err := validateKey(key)
if err != nil {
return err
}
helpers, err := sysregistriesv2.CredentialHelpers(sys)
if err != nil {
return err
@ -316,17 +364,22 @@ func RemoveAuthentication(sys *types.SystemContext, registry string) error {
isLoggedIn := false
removeFromCredHelper := func(helper string) {
err := deleteAuthFromCredHelper(helper, registry)
if err == nil {
logrus.Debugf("Credentials for %q were deleted from credential helper %s", registry, helper)
isLoggedIn = true
if isNamespaced {
logrus.Debugf("Not removing credentials because namespaced keys are not supported for the credential helper: %s", helper)
return
} else {
err := deleteAuthFromCredHelper(helper, key)
if err == nil {
logrus.Debugf("Credentials for %q were deleted from credential helper %s", key, helper)
isLoggedIn = true
return
}
if credentials.IsErrCredentialsNotFoundMessage(err.Error()) {
logrus.Debugf("Not logged in to %s with credential helper %s", key, helper)
return
}
}
if credentials.IsErrCredentialsNotFoundMessage(err.Error()) {
logrus.Debugf("Not logged in to %s with credential helper %s", registry, helper)
return
}
multiErr = multierror.Append(multiErr, errors.Wrapf(err, "error removing credentials for %s from credential helper %s", registry, helper))
multiErr = multierror.Append(multiErr, errors.Wrapf(err, "removing credentials for %s from credential helper %s", key, helper))
}
for _, helper := range helpers {
@ -335,15 +388,12 @@ func RemoveAuthentication(sys *types.SystemContext, registry string) error {
// Special-case the built-in helper for auth files.
case sysregistriesv2.AuthenticationFileHelper:
_, err = modifyJSON(sys, func(auths *dockerConfigFile) (bool, error) {
if innerHelper, exists := auths.CredHelpers[registry]; exists {
if innerHelper, exists := auths.CredHelpers[key]; exists {
removeFromCredHelper(innerHelper)
}
if _, ok := auths.AuthConfigs[registry]; ok {
if _, ok := auths.AuthConfigs[key]; ok {
isLoggedIn = true
delete(auths.AuthConfigs, registry)
} else if _, ok := auths.AuthConfigs[normalizeRegistry(registry)]; ok {
isLoggedIn = true
delete(auths.AuthConfigs, normalizeRegistry(registry))
delete(auths.AuthConfigs, key)
}
return true, multiErr
})
@ -486,13 +536,13 @@ func readJSONFile(path string, legacyFormat bool) (dockerConfigFile, error) {
if legacyFormat {
if err = json.Unmarshal(raw, &auths.AuthConfigs); err != nil {
return dockerConfigFile{}, errors.Wrapf(err, "error unmarshaling JSON at %q", path)
return dockerConfigFile{}, errors.Wrapf(err, "unmarshaling JSON at %q", path)
}
return auths, nil
}
if err = json.Unmarshal(raw, &auths); err != nil {
return dockerConfigFile{}, errors.Wrapf(err, "error unmarshaling JSON at %q", path)
return dockerConfigFile{}, errors.Wrapf(err, "unmarshaling JSON at %q", path)
}
if auths.AuthConfigs == nil {
@ -524,21 +574,21 @@ func modifyJSON(sys *types.SystemContext, editor func(auths *dockerConfigFile) (
auths, err := readJSONFile(path, false)
if err != nil {
return "", errors.Wrapf(err, "error reading JSON file %q", path)
return "", errors.Wrapf(err, "reading JSON file %q", path)
}
updated, err := editor(&auths)
if err != nil {
return "", errors.Wrapf(err, "error updating %q", path)
return "", errors.Wrapf(err, "updating %q", path)
}
if updated {
newData, err := json.MarshalIndent(auths, "", "\t")
if err != nil {
return "", errors.Wrapf(err, "error marshaling JSON %q", path)
return "", errors.Wrapf(err, "marshaling JSON %q", path)
}
if err = ioutil.WriteFile(path, newData, 0600); err != nil {
return "", errors.Wrapf(err, "error writing to file %q", path)
return "", errors.Wrapf(err, "writing to file %q", path)
}
}
@ -575,11 +625,13 @@ func deleteAuthFromCredHelper(credHelper, registry string) error {
return helperclient.Erase(p, registry)
}
// findAuthentication looks for auth of registry in path
func findAuthentication(registry, path string, legacyFormat bool) (types.DockerAuthConfig, error) {
// findAuthentication looks for auth of registry in path. If ref is
// not nil, then it will be taken into account when looking up the
// authentication credentials.
func findAuthentication(ref reference.Named, registry, path string, legacyFormat bool) (types.DockerAuthConfig, error) {
auths, err := readJSONFile(path, legacyFormat)
if err != nil {
return types.DockerAuthConfig{}, errors.Wrapf(err, "error reading JSON file %q", path)
return types.DockerAuthConfig{}, errors.Wrapf(err, "reading JSON file %q", path)
}
// First try cred helpers. They should always be normalized.
@ -587,16 +639,36 @@ func findAuthentication(registry, path string, legacyFormat bool) (types.DockerA
return getAuthFromCredHelper(ch, registry)
}
// I'm feeling lucky
if val, exists := auths.AuthConfigs[registry]; exists {
return decodeDockerAuth(val)
// Support for different paths in auth.
// (This is not a feature of ~/.docker/config.json; we support it even for
// those files as an extension.)
var keys []string
if !legacyFormat && ref != nil {
keys = authKeysForRef(ref)
} else {
keys = []string{registry}
}
// Repo or namespace keys are only supported as exact matches. For registry
// keys we prefer exact matches as well.
for _, key := range keys {
if val, exists := auths.AuthConfigs[key]; exists {
return decodeDockerAuth(val)
}
}
// bad luck; let's normalize the entries first
registry = normalizeRegistry(registry)
// This primarily happens for legacyFormat, which for a time used API URLs
// (http[s:]//…/v1/) as keys.
// Secondarily, (docker login) accepted URLs with no normalization for
// several years, and matched registry hostnames against that, so support
// those entries even in non-legacyFormat ~/.docker/config.json.
// The docker.io registry still uses the /v1/ key with a special host name,
// so account for that as well.
registry = normalizeAuthFileKey(registry, legacyFormat)
normalizedAuths := map[string]dockerAuthConfig{}
for k, v := range auths.AuthConfigs {
normalizedAuths[normalizeRegistry(k)] = v
normalizedAuths[normalizeAuthFileKey(k, legacyFormat)] = v
}
if val, exists := normalizedAuths[registry]; exists {
@ -606,6 +678,28 @@ func findAuthentication(registry, path string, legacyFormat bool) (types.DockerA
return types.DockerAuthConfig{}, nil
}
// authKeysForRef returns the valid paths for a provided reference. For example,
// when given a reference "quay.io/repo/ns/image:tag", then it would return
// - quay.io/repo/ns/image
// - quay.io/repo/ns
// - quay.io/repo
// - quay.io
func authKeysForRef(ref reference.Named) (res []string) {
name := ref.Name()
for {
res = append(res, name)
lastSlash := strings.LastIndex(name, "/")
if lastSlash == -1 {
break
}
name = name[:lastSlash]
}
return res
}
// decodeDockerAuth decodes the username and password, which is
// encoded in base64.
func decodeDockerAuth(conf dockerAuthConfig) (types.DockerAuthConfig, error) {
@ -629,27 +723,36 @@ func decodeDockerAuth(conf dockerAuthConfig) (types.DockerAuthConfig, error) {
}, nil
}
// convertToHostname converts a registry url which has http|https prepended
// to just an hostname.
// Copied from github.com/docker/docker/registry/auth.go
func convertToHostname(url string) string {
stripped := url
if strings.HasPrefix(url, "http://") {
stripped = strings.TrimPrefix(url, "http://")
} else if strings.HasPrefix(url, "https://") {
stripped = strings.TrimPrefix(url, "https://")
// normalizeAuthFileKey takes a key, converts it to a host name and normalizes
// the resulting registry.
func normalizeAuthFileKey(key string, legacyFormat bool) string {
stripped := strings.TrimPrefix(key, "http://")
stripped = strings.TrimPrefix(stripped, "https://")
if legacyFormat || stripped != key {
stripped = strings.SplitN(stripped, "/", 2)[0]
}
nameParts := strings.SplitN(stripped, "/", 2)
return nameParts[0]
return normalizeRegistry(stripped)
}
// normalizeRegistry converts the provided registry if a known docker.io host
// is provided.
func normalizeRegistry(registry string) string {
normalized := convertToHostname(registry)
switch normalized {
switch registry {
case "registry-1.docker.io", "docker.io":
return "index.docker.io"
}
return normalized
return registry
}
// validateKey verifies that the input key does not have a prefix that is not
// allowed and returns an indicator if the key is namespaced.
func validateKey(key string) (isNamespaced bool, err error) {
if strings.HasPrefix(key, "http://") || strings.HasPrefix(key, "https://") {
return isNamespaced, errors.Errorf("key %s contains http[s]:// prefix", key)
}
// check if the provided key contains one or more subpaths.
return strings.ContainsRune(key, '/'), nil
}

8
vendor/github.com/containers/image/v5/pkg/docker/config/config_linux.go generated vendored
View File

@ -73,7 +73,7 @@ func removeAllAuthFromKernelKeyring() error { //nolint:deadcode,unused
if strings.HasPrefix(keyDescribe, keyDescribePrefix) {
err := keyctl.Unlink(userkeyring, k)
if err != nil {
return errors.Wrapf(err, "error unlinking key %d", k.ID())
return errors.Wrapf(err, "unlinking key %d", k.ID())
}
logrus.Debugf("unlinked key %d:%s", k.ID(), keyAttr)
}
@ -100,16 +100,16 @@ func setAuthToKernelKeyring(registry, username, password string) error { //nolin
// link the key to userKeyring
userKeyring, err := keyctl.UserKeyring()
if err != nil {
return errors.Wrapf(err, "error getting user keyring")
return errors.Wrapf(err, "getting user keyring")
}
err = keyctl.Link(userKeyring, id)
if err != nil {
return errors.Wrapf(err, "error linking the key to user keyring")
return errors.Wrapf(err, "linking the key to user keyring")
}
// unlink the key from session keyring
err = keyctl.Unlink(keyring, id)
if err != nil {
return errors.Wrapf(err, "error unlinking the key from session keyring")
return errors.Wrapf(err, "unlinking the key from session keyring")
}
return nil
}

6
vendor/github.com/containers/image/v5/pkg/shortnames/shortnames.go generated vendored
View File

@ -211,7 +211,7 @@ func (c *PullCandidate) Record() error {
value := reference.TrimNamed(c.Value)
if err := Add(c.resolved.systemContext, name.String(), value); err != nil {
return errors.Wrapf(err, "error recording short-name alias (%q=%q)", c.resolved.userInput, c.Value)
return errors.Wrapf(err, "recording short-name alias (%q=%q)", c.resolved.userInput, c.Value)
}
return nil
}
@ -323,7 +323,7 @@ func Resolve(ctx *types.SystemContext, name string) (*Resolved, error) {
for _, reg := range unqualifiedSearchRegistries {
named, err := reference.ParseNormalizedNamed(fmt.Sprintf("%s/%s", reg, name))
if err != nil {
return nil, errors.Wrapf(err, "error creating reference with unqualified-search registry %q", reg)
return nil, errors.Wrapf(err, "creating reference with unqualified-search registry %q", reg)
}
// Make sure to add ":latest" if needed
named = reference.TagNameOnly(named)
@ -450,7 +450,7 @@ func ResolveLocally(ctx *types.SystemContext, name string) ([]reference.Named, e
for _, reg := range append([]string{"localhost"}, unqualifiedSearchRegistries...) {
named, err := reference.ParseNormalizedNamed(fmt.Sprintf("%s/%s", reg, name))
if err != nil {
return nil, errors.Wrapf(err, "error creating reference with unqualified-search registry %q", reg)
return nil, errors.Wrapf(err, "creating reference with unqualified-search registry %q", reg)
}
// Make sure to add ":latest" if needed
named = reference.TagNameOnly(named)

18
vendor/github.com/containers/image/v5/pkg/sysregistriesv2/shortnames.go generated vendored
View File

@ -3,6 +3,7 @@ package sysregistriesv2
import (
"os"
"path/filepath"
"reflect"
"strings"
"github.com/BurntSushi/toml"
@ -49,6 +50,17 @@ type shortNameAliasConf struct {
// reference counter parts.
// Note that Aliases is niled after being loaded from a file.
Aliases map[string]string `toml:"aliases"`
// If you add any field, make sure to update nonempty() below.
}
// nonempty returns true if config contains at least one configuration entry.
func (c *shortNameAliasConf) nonempty() bool {
copy := *c // A shallow copy
if copy.Aliases != nil && len(copy.Aliases) == 0 {
copy.Aliases = nil
}
return !reflect.DeepEqual(copy, shortNameAliasConf{})
}
// alias combines the parsed value of an alias with the config file it has been
@ -197,7 +209,7 @@ func RemoveShortNameAlias(ctx *types.SystemContext, name string) error {
func parseShortNameValue(alias string) (reference.Named, error) {
ref, err := reference.Parse(alias)
if err != nil {
return nil, errors.Wrapf(err, "error parsing alias %q", alias)
return nil, errors.Wrapf(err, "parsing alias %q", alias)
}
if _, ok := ref.(reference.Digested); ok {
@ -306,14 +318,14 @@ func loadShortNameAliasConf(confPath string) (*shortNameAliasConf, *shortNameAli
_, err := toml.DecodeFile(confPath, &conf)
if err != nil && !os.IsNotExist(err) {
// It's okay if the config doesn't exist. Other errors are not.
return nil, nil, errors.Wrapf(err, "error loading short-name aliases config file %q", confPath)
return nil, nil, errors.Wrapf(err, "loading short-name aliases config file %q", confPath)
}
// Even if we don’t always need the cache, doing so validates the machine-generated config. The
// file could still be corrupted by another process or user.
cache, err := newShortNameAliasCache(confPath, &conf)
if err != nil {
return nil, nil, errors.Wrapf(err, "error loading short-name aliases config file %q", confPath)
return nil, nil, errors.Wrapf(err, "loading short-name aliases config file %q", confPath)
}
return &conf, cache, nil

43
vendor/github.com/containers/image/v5/pkg/sysregistriesv2/system_registries_v2.go generated vendored
View File

@ -4,6 +4,7 @@ import (
"fmt"
"os"
"path/filepath"
"reflect"
"regexp"
"sort"
"strings"
@ -87,7 +88,7 @@ func (e *Endpoint) rewriteReference(ref reference.Named, prefix string) (referen
newNamedRef = e.Location + refString[prefixLen:]
newParsedRef, err := reference.ParseNamed(newNamedRef)
if err != nil {
return nil, errors.Wrapf(err, "error rewriting reference")
return nil, errors.Wrapf(err, "rewriting reference")
}
return newParsedRef, nil
@ -172,9 +173,17 @@ type V1RegistriesConf struct {
// Nonempty returns true if config contains at least one configuration entry.
func (config *V1RegistriesConf) Nonempty() bool {
return (len(config.V1TOMLConfig.Search.Registries) != 0 ||
len(config.V1TOMLConfig.Insecure.Registries) != 0 ||
len(config.V1TOMLConfig.Block.Registries) != 0)
copy := *config // A shallow copy
if copy.V1TOMLConfig.Search.Registries != nil && len(copy.V1TOMLConfig.Search.Registries) == 0 {
copy.V1TOMLConfig.Search.Registries = nil
}
if copy.V1TOMLConfig.Insecure.Registries != nil && len(copy.V1TOMLConfig.Insecure.Registries) == 0 {
copy.V1TOMLConfig.Insecure.Registries = nil
}
if copy.V1TOMLConfig.Block.Registries != nil && len(copy.V1TOMLConfig.Block.Registries) == 0 {
copy.V1TOMLConfig.Block.Registries = nil
}
return !reflect.DeepEqual(copy, V1RegistriesConf{})
}
// V2RegistriesConf is the sysregistries v2 configuration format.
@ -203,12 +212,26 @@ type V2RegistriesConf struct {
ShortNameMode string `toml:"short-name-mode"`
shortNameAliasConf
// If you add any field, make sure to update Nonempty() below.
}
// Nonempty returns true if config contains at least one configuration entry.
func (config *V2RegistriesConf) Nonempty() bool {
return (len(config.Registries) != 0 ||
len(config.UnqualifiedSearchRegistries) != 0)
copy := *config // A shallow copy
if copy.Registries != nil && len(copy.Registries) == 0 {
copy.Registries = nil
}
if copy.UnqualifiedSearchRegistries != nil && len(copy.UnqualifiedSearchRegistries) == 0 {
copy.UnqualifiedSearchRegistries = nil
}
if copy.CredentialHelpers != nil && len(copy.CredentialHelpers) == 0 {
copy.CredentialHelpers = nil
}
if !copy.shortNameAliasConf.nonempty() {
copy.shortNameAliasConf = shortNameAliasConf{}
}
return !reflect.DeepEqual(copy, V2RegistriesConf{})
}
// parsedConfig is the result of parsing, and possibly merging, configuration files;
@ -604,7 +627,7 @@ func dropInConfigs(wrapper configWrapper) ([]string, error) {
if err != nil && !os.IsNotExist(err) {
// Ignore IsNotExist errors: most systems won't have a registries.conf.d
// directory.
return nil, errors.Wrapf(err, "error reading registries.conf.d")
return nil, errors.Wrapf(err, "reading registries.conf.d")
}
}
@ -646,7 +669,7 @@ func tryUpdatingCache(ctx *types.SystemContext, wrapper configWrapper) (*parsedC
return nil, err // Should never happen
}
} else {
return nil, errors.Wrapf(err, "error loading registries configuration %q", wrapper.configPath)
return nil, errors.Wrapf(err, "loading registries configuration %q", wrapper.configPath)
}
}
@ -659,7 +682,7 @@ func tryUpdatingCache(ctx *types.SystemContext, wrapper configWrapper) (*parsedC
// Enforce v2 format for drop-in-configs.
dropIn, err := loadConfigFile(path, true)
if err != nil {
return nil, errors.Wrapf(err, "error loading drop-in registries configuration %q", path)
return nil, errors.Wrapf(err, "loading drop-in registries configuration %q", path)
}
config.updateWithConfigurationFrom(dropIn)
}
@ -910,7 +933,7 @@ func loadConfigFile(path string, forceV2 bool) (*parsedConfig, error) {
// Parse and validate short-name aliases.
cache, err := newShortNameAliasCache(path, &res.partialV2.shortNameAliasConf)
if err != nil {
return nil, errors.Wrap(err, "error validating short-name aliases")
return nil, errors.Wrap(err, "validating short-name aliases")
}
res.aliasCache = cache
// Clear conf.partialV2.shortNameAliasConf to make it available for garbage collection and

257
vendor/github.com/containers/image/v5/storage/storage_image.go generated vendored
View File

@ -23,7 +23,9 @@ import (
"github.com/containers/image/v5/pkg/blobinfocache/none"
"github.com/containers/image/v5/types"
"github.com/containers/storage"
"github.com/containers/storage/drivers"
"github.com/containers/storage/pkg/archive"
"github.com/containers/storage/pkg/chunked"
"github.com/containers/storage/pkg/ioutils"
digest "github.com/opencontainers/go-digest"
imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1"
@ -59,6 +61,7 @@ type storageImageDestination struct {
directory string // Temporary directory where we store blobs until Commit() time
nextTempFileID int32 // A counter that we use for computing filenames to assign to blobs
manifest []byte // Manifest contents, temporary
manifestDigest digest.Digest // Valid if len(manifest) != 0
signatures []byte // Signature contents, temporary
signatureses map[digest.Digest][]byte // Instance signature contents, temporary
SignatureSizes []int `json:"signature-sizes,omitempty"` // List of sizes of each signature slice
@ -76,12 +79,13 @@ type storageImageDestination struct {
indexToStorageID map[int]*string
// All accesses to below data are protected by `lock` which is made
// *explicit* in the code.
blobDiffIDs map[digest.Digest]digest.Digest // Mapping from layer blobsums to their corresponding DiffIDs
fileSizes map[digest.Digest]int64 // Mapping from layer blobsums to their sizes
filenames map[digest.Digest]string // Mapping from layer blobsums to names of files we used to hold them
currentIndex int // The index of the layer to be committed (i.e., lower indices have already been committed)
indexToPulledLayerInfo map[int]*manifest.LayerInfo // Mapping from layer (by index) to pulled down blob
blobAdditionalLayer map[digest.Digest]storage.AdditionalLayer // Mapping from layer blobsums to their corresponding additional layer
blobDiffIDs map[digest.Digest]digest.Digest // Mapping from layer blobsums to their corresponding DiffIDs
fileSizes map[digest.Digest]int64 // Mapping from layer blobsums to their sizes
filenames map[digest.Digest]string // Mapping from layer blobsums to names of files we used to hold them
currentIndex int // The index of the layer to be committed (i.e., lower indices have already been committed)
indexToPulledLayerInfo map[int]*manifest.LayerInfo // Mapping from layer (by index) to pulled down blob
blobAdditionalLayer map[digest.Digest]storage.AdditionalLayer // Mapping from layer blobsums to their corresponding additional layer
diffOutputs map[digest.Digest]*graphdriver.DriverWithDifferOutput // Mapping from digest to differ output
}
type storageImageCloser struct {
@ -121,7 +125,7 @@ func newImageSource(ctx context.Context, sys *types.SystemContext, imageRef stor
}
if img.Metadata != "" {
if err := json.Unmarshal([]byte(img.Metadata), image); err != nil {
return nil, errors.Wrap(err, "error decoding metadata for source image")
return nil, errors.Wrap(err, "decoding metadata for source image")
}
}
return image, nil
@ -239,7 +243,7 @@ func (s *storageImageSource) GetManifest(ctx context.Context, instanceDigest *di
key := manifestBigDataKey(*instanceDigest)
blob, err := s.imageRef.transport.store.ImageBigData(s.image.ID, key)
if err != nil {
return nil, "", errors.Wrapf(err, "error reading manifest for image instance %q", *instanceDigest)
return nil, "", errors.Wrapf(err, "reading manifest for image instance %q", *instanceDigest)
}
return blob, manifest.GuessMIMEType(blob), err
}
@ -276,14 +280,14 @@ func (s *storageImageSource) GetManifest(ctx context.Context, instanceDigest *di
func (s *storageImageSource) LayerInfosForCopy(ctx context.Context, instanceDigest *digest.Digest) ([]types.BlobInfo, error) {
manifestBlob, manifestType, err := s.GetManifest(ctx, instanceDigest)
if err != nil {
return nil, errors.Wrapf(err, "error reading image manifest for %q", s.image.ID)
return nil, errors.Wrapf(err, "reading image manifest for %q", s.image.ID)
}
if manifest.MIMETypeIsMultiImage(manifestType) {
return nil, errors.Errorf("can't copy layers for a manifest list (shouldn't be attempted)")
}
man, err := manifest.FromBlob(manifestBlob, manifestType)
if err != nil {
return nil, errors.Wrapf(err, "error parsing image manifest for %q", s.image.ID)
return nil, errors.Wrapf(err, "parsing image manifest for %q", s.image.ID)
}
uncompressedLayerType := ""
@ -299,7 +303,7 @@ func (s *storageImageSource) LayerInfosForCopy(ctx context.Context, instanceDige
for layerID != "" {
layer, err := s.imageRef.transport.store.Layer(layerID)
if err != nil {
return nil, errors.Wrapf(err, "error reading layer %q in image %q", layerID, s.image.ID)
return nil, errors.Wrapf(err, "reading layer %q in image %q", layerID, s.image.ID)
}
if layer.UncompressedDigest == "" {
return nil, errors.Errorf("uncompressed digest for layer %q is unknown", layerID)
@ -318,7 +322,7 @@ func (s *storageImageSource) LayerInfosForCopy(ctx context.Context, instanceDige
res, err := buildLayerInfosForCopy(man.LayerInfos(), physicalBlobInfos)
if err != nil {
return nil, errors.Wrapf(err, "error creating LayerInfosForCopy of image %q", s.image.ID)
return nil, errors.Wrapf(err, "creating LayerInfosForCopy of image %q", s.image.ID)
}
return res, nil
}
@ -367,13 +371,13 @@ func (s *storageImageSource) GetSignatures(ctx context.Context, instanceDigest *
if len(signatureSizes) > 0 {
signatureBlob, err := s.imageRef.transport.store.ImageBigData(s.image.ID, key)
if err != nil {
return nil, errors.Wrapf(err, "error looking up signatures data for image %q (%s)", s.image.ID, instance)
return nil, errors.Wrapf(err, "looking up signatures data for image %q (%s)", s.image.ID, instance)
}
signature = signatureBlob
}
for _, length := range signatureSizes {
if offset+length > len(signature) {
return nil, errors.Wrapf(err, "error looking up signatures data for image %q (%s): expected at least %d bytes, only found %d", s.image.ID, instance, len(signature), offset+length)
return nil, errors.Wrapf(err, "looking up signatures data for image %q (%s): expected at least %d bytes, only found %d", s.image.ID, instance, len(signature), offset+length)
}
sigslice = append(sigslice, signature[offset:offset+length])
offset += length
@ -389,7 +393,7 @@ func (s *storageImageSource) GetSignatures(ctx context.Context, instanceDigest *
func newImageDestination(sys *types.SystemContext, imageRef storageReference) (*storageImageDestination, error) {
directory, err := ioutil.TempDir(tmpdir.TemporaryDirectoryForBigFiles(sys), "storage")
if err != nil {
return nil, errors.Wrapf(err, "error creating a temporary directory")
return nil, errors.Wrapf(err, "creating a temporary directory")
}
image := &storageImageDestination{
imageRef: imageRef,
@ -403,6 +407,7 @@ func newImageDestination(sys *types.SystemContext, imageRef storageReference) (*
SignaturesSizes: make(map[digest.Digest][]int),
indexToStorageID: make(map[int]*string),
indexToPulledLayerInfo: make(map[int]*manifest.LayerInfo),
diffOutputs: make(map[digest.Digest]*graphdriver.DriverWithDifferOutput),
}
return image, nil
}
@ -418,6 +423,11 @@ func (s *storageImageDestination) Close() error {
for _, al := range s.blobAdditionalLayer {
al.Release()
}
for _, v := range s.diffOutputs {
if v.Target != "" {
_ = s.imageRef.transport.store.CleanupStagingDirectory(v.Target)
}
}
return os.RemoveAll(s.directory)
}
@ -483,21 +493,21 @@ func (s *storageImageDestination) PutBlob(ctx context.Context, stream io.Reader,
filename := s.computeNextBlobCacheFile()
file, err := os.OpenFile(filename, os.O_CREATE|os.O_TRUNC|os.O_WRONLY|os.O_EXCL, 0600)
if err != nil {
return errorBlobInfo, errors.Wrapf(err, "error creating temporary file %q", filename)
return errorBlobInfo, errors.Wrapf(err, "creating temporary file %q", filename)
}
defer file.Close()
counter := ioutils.NewWriteCounter(hasher.Hash())
reader := io.TeeReader(io.TeeReader(stream, counter), file)
decompressed, err := archive.DecompressStream(reader)
if err != nil {
return errorBlobInfo, errors.Wrap(err, "error setting up to decompress blob")
return errorBlobInfo, errors.Wrap(err, "setting up to decompress blob")
}
// Copy the data to the file.
// TODO: This can take quite some time, and should ideally be cancellable using ctx.Done().
_, err = io.Copy(diffID.Hash(), decompressed)
decompressed.Close()
if err != nil {
return errorBlobInfo, errors.Wrapf(err, "error storing blob to file %q", filename)
return errorBlobInfo, errors.Wrapf(err, "storing blob to file %q", filename)
}
// Ensure that any information that we were given about the blob is correct.
if blobinfo.Digest.Validate() == nil && blobinfo.Digest != hasher.Digest() {
@ -556,7 +566,7 @@ func (s *storageImageDestination) tryReusingBlobWithSrcRef(ctx context.Context,
// Check if we have the layer in the underlying additional layer store.
aLayer, err := s.imageRef.transport.store.LookupAdditionalLayer(blobinfo.Digest, ref.String())
if err != nil && errors.Cause(err) != storage.ErrLayerUnknown {
return false, types.BlobInfo{}, errors.Wrapf(err, `Error looking for compressed layers with digest %q and labels`, blobinfo.Digest)
return false, types.BlobInfo{}, errors.Wrapf(err, `looking for compressed layers with digest %q and labels`, blobinfo.Digest)
} else if err == nil {
// Record the uncompressed value so that we can use it to calculate layer IDs.
s.blobDiffIDs[blobinfo.Digest] = aLayer.UncompressedDigest()
@ -572,6 +582,61 @@ func (s *storageImageDestination) tryReusingBlobWithSrcRef(ctx context.Context,
return s.tryReusingBlobLocked(ctx, blobinfo, cache, canSubstitute)
}
type zstdFetcher struct {
stream internalTypes.ImageSourceSeekable
ctx context.Context
blobInfo types.BlobInfo
}
// GetBlobAt converts from chunked.GetBlobAt to ImageSourceSeekable.GetBlobAt.
func (f *zstdFetcher) GetBlobAt(chunks []chunked.ImageSourceChunk) (chan io.ReadCloser, chan error, error) {
var newChunks []internalTypes.ImageSourceChunk
for _, v := range chunks {
i := internalTypes.ImageSourceChunk{
Offset: v.Offset,
Length: v.Length,
}
newChunks = append(newChunks, i)
}
rc, errs, err := f.stream.GetBlobAt(f.ctx, f.blobInfo, newChunks)
if _, ok := err.(internalTypes.BadPartialRequestError); ok {
err = chunked.ErrBadRequest{}
}
return rc, errs, err
}
// PutBlobPartial attempts to create a blob using the data that is already present at the destination storage. stream is accessed
// in a non-sequential way to retrieve the missing chunks.
func (s *storageImageDestination) PutBlobPartial(ctx context.Context, stream internalTypes.ImageSourceSeekable, srcInfo types.BlobInfo, cache types.BlobInfoCache) (types.BlobInfo, error) {
fetcher := zstdFetcher{
stream: stream,
ctx: ctx,
blobInfo: srcInfo,
}
differ, err := chunked.GetDiffer(ctx, s.imageRef.transport.store, srcInfo.Size, srcInfo.Annotations, &fetcher)
if err != nil {
return srcInfo, err
}
out, err := s.imageRef.transport.store.ApplyDiffWithDiffer("", nil, differ)
if err != nil {
return srcInfo, err
}
blobDigest := srcInfo.Digest
s.lock.Lock()
s.blobDiffIDs[blobDigest] = blobDigest
s.fileSizes[blobDigest] = 0
s.filenames[blobDigest] = ""
s.diffOutputs[blobDigest] = out
s.lock.Unlock()
return srcInfo, nil
}
// TryReusingBlob checks whether the transport already contains, or can efficiently reuse, a blob, and if so, applies it to the current destination
// (e.g. if the blob is a filesystem layer, this signifies that the changes it describes need to be applied again when composing a filesystem tree).
// info.Digest must not be empty.
@ -611,7 +676,7 @@ func (s *storageImageDestination) tryReusingBlobLocked(ctx context.Context, blob
// Check if we have a wasn't-compressed layer in storage that's based on that blob.
layers, err := s.imageRef.transport.store.LayersByUncompressedDigest(blobinfo.Digest)
if err != nil && errors.Cause(err) != storage.ErrLayerUnknown {
return false, types.BlobInfo{}, errors.Wrapf(err, `Error looking for layers with digest %q`, blobinfo.Digest)
return false, types.BlobInfo{}, errors.Wrapf(err, `looking for layers with digest %q`, blobinfo.Digest)
}
if len(layers) > 0 {
// Save this for completeness.
@ -626,7 +691,7 @@ func (s *storageImageDestination) tryReusingBlobLocked(ctx context.Context, blob
// Check if we have a was-compressed layer in storage that's based on that blob.
layers, err = s.imageRef.transport.store.LayersByCompressedDigest(blobinfo.Digest)
if err != nil && errors.Cause(err) != storage.ErrLayerUnknown {
return false, types.BlobInfo{}, errors.Wrapf(err, `Error looking for compressed layers with digest %q`, blobinfo.Digest)
return false, types.BlobInfo{}, errors.Wrapf(err, `looking for compressed layers with digest %q`, blobinfo.Digest)
}
if len(layers) > 0 {
// Record the uncompressed value so that we can use it to calculate layer IDs.
@ -645,7 +710,7 @@ func (s *storageImageDestination) tryReusingBlobLocked(ctx context.Context, blob
if uncompressedDigest := cache.UncompressedDigest(blobinfo.Digest); uncompressedDigest != "" && uncompressedDigest != blobinfo.Digest {
layers, err := s.imageRef.transport.store.LayersByUncompressedDigest(uncompressedDigest)
if err != nil && errors.Cause(err) != storage.ErrLayerUnknown {
return false, types.BlobInfo{}, errors.Wrapf(err, `Error looking for layers with digest %q`, uncompressedDigest)
return false, types.BlobInfo{}, errors.Wrapf(err, `looking for layers with digest %q`, uncompressedDigest)
}
if len(layers) > 0 {
if blobinfo.Size != -1 {
@ -720,7 +785,7 @@ func (s *storageImageDestination) getConfigBlob(info types.BlobInfo) ([]byte, er
if filename, ok := s.filenames[info.Digest]; ok {
contents, err2 := ioutil.ReadFile(filename)
if err2 != nil {
return nil, errors.Wrapf(err2, `error reading blob from file %q`, filename)
return nil, errors.Wrapf(err2, `reading blob from file %q`, filename)
}
return contents, nil
}
@ -822,7 +887,7 @@ func (s *storageImageDestination) commitLayer(ctx context.Context, blob manifest
// NOTE: use `TryReusingBlob` to prevent recursion.
has, _, err := s.TryReusingBlob(ctx, blob.BlobInfo, none.NoCache, false)
if err != nil {
return errors.Wrapf(err, "error checking for a layer based on blob %q", blob.Digest.String())
return errors.Wrapf(err, "checking for a layer based on blob %q", blob.Digest.String())
}
if !has {
return errors.Errorf("error determining uncompressed digest for blob %q", blob.Digest.String())
@ -843,6 +908,27 @@ func (s *storageImageDestination) commitLayer(ctx context.Context, blob manifest
return nil
}
s.lock.Lock()
diffOutput, ok := s.diffOutputs[blob.Digest]
s.lock.Unlock()
if ok {
layer, err := s.imageRef.transport.store.CreateLayer(id, lastLayer, nil, "", false, nil)
if err != nil {
return err
}
// FIXME: what to do with the uncompressed digest?
diffOutput.UncompressedDigest = blob.Digest
if err := s.imageRef.transport.store.ApplyDiffFromStagingDirectory(layer.ID, diffOutput.Target, diffOutput, nil); err != nil {
_ = s.imageRef.transport.store.Delete(layer.ID)
return err
}
s.indexToStorageID[index] = &layer.ID
return nil
}
s.lock.Lock()
al, ok := s.blobAdditionalLayer[blob.Digest]
s.lock.Unlock()
@ -874,7 +960,7 @@ func (s *storageImageDestination) commitLayer(ctx context.Context, blob manifest
}
}
if layer == "" {
return errors.Wrapf(err2, "error locating layer for blob %q", blob.Digest)
return errors.Wrapf(err2, "locating layer for blob %q", blob.Digest)
}
// Read the layer's contents.
noCompression := archive.Uncompressed
@ -883,7 +969,7 @@ func (s *storageImageDestination) commitLayer(ctx context.Context, blob manifest
}
diff, err2 := s.imageRef.transport.store.Diff("", layer, diffOptions)
if err2 != nil {
return errors.Wrapf(err2, "error reading layer %q for blob %q", layer, blob.Digest)
return errors.Wrapf(err2, "reading layer %q for blob %q", layer, blob.Digest)
}
// Copy the layer diff to a file. Diff() takes a lock that it holds
// until the ReadCloser that it returns is closed, and PutLayer() wants
@ -893,7 +979,7 @@ func (s *storageImageDestination) commitLayer(ctx context.Context, blob manifest
file, err := os.OpenFile(filename, os.O_CREATE|os.O_TRUNC|os.O_WRONLY|os.O_EXCL, 0600)
if err != nil {
diff.Close()
return errors.Wrapf(err, "error creating temporary file %q", filename)
return errors.Wrapf(err, "creating temporary file %q", filename)
}
// Copy the data to the file.
// TODO: This can take quite some time, and should ideally be cancellable using
@ -902,7 +988,7 @@ func (s *storageImageDestination) commitLayer(ctx context.Context, blob manifest
diff.Close()
file.Close()
if err != nil {
return errors.Wrapf(err, "error storing blob to file %q", filename)
return errors.Wrapf(err, "storing blob to file %q", filename)
}
// Make sure that we can find this file later, should we need the layer's
// contents again.
@ -913,27 +999,34 @@ func (s *storageImageDestination) commitLayer(ctx context.Context, blob manifest
// Read the cached blob and use it as a diff.
file, err := os.Open(filename)
if err != nil {
return errors.Wrapf(err, "error opening file %q", filename)
return errors.Wrapf(err, "opening file %q", filename)
}
defer file.Close()
// Build the new layer using the diff, regardless of where it came from.
// TODO: This can take quite some time, and should ideally be cancellable using ctx.Done().
layer, _, err := s.imageRef.transport.store.PutLayer(id, lastLayer, nil, "", false, nil, file)
if err != nil && errors.Cause(err) != storage.ErrDuplicateID {
return errors.Wrapf(err, "error adding layer with blob %q", blob.Digest)
return errors.Wrapf(err, "adding layer with blob %q", blob.Digest)
}
s.indexToStorageID[index] = &layer.ID
return nil
}
// Commit marks the process of storing the image as successful and asks for the image to be persisted.
// unparsedToplevel contains data about the top-level manifest of the source (which may be a single-arch image or a manifest list
// if PutManifest was only called for the single-arch image with instanceDigest == nil), primarily to allow lookups by the
// original manifest list digest, if desired.
// WARNING: This does not have any transactional semantics:
// - Uploaded data MAY be visible to others before Commit() is called
// - Uploaded data MAY be removed or MAY remain around if Close() is called without Commit() (i.e. rollback is allowed but not guaranteed)
func (s *storageImageDestination) Commit(ctx context.Context, unparsedToplevel types.UnparsedImage) error {
if len(s.manifest) == 0 {
return errors.New("Internal error: storageImageDestination.Commit() called without PutManifest()")
}
toplevelManifest, _, err := unparsedToplevel.Manifest(ctx)
if err != nil {
return errors.Wrapf(err, "error retrieving top-level manifest")
return errors.Wrapf(err, "retrieving top-level manifest")
}
// If the name we're saving to includes a digest, then check that the
// manifests that we're about to save all either match the one from the
@ -956,14 +1049,12 @@ func (s *storageImageDestination) Commit(ctx context.Context, unparsedToplevel t
}
}
// Find the list of layer blobs.
if len(s.manifest) == 0 {
return errors.New("Internal error: storageImageDestination.Commit() called without PutManifest()")
}
man, err := manifest.FromBlob(s.manifest, manifest.GuessMIMEType(s.manifest))
if err != nil {
return errors.Wrapf(err, "error parsing manifest")
return errors.Wrapf(err, "parsing manifest")
}
layerBlobs := man.LayerInfos()
// Extract, commit, or find the layers.
for i, blob := range layerBlobs {
if err := s.commitLayer(ctx, blob, i); err != nil {
@ -996,11 +1087,11 @@ func (s *storageImageDestination) Commit(ctx context.Context, unparsedToplevel t
if err != nil {
if errors.Cause(err) != storage.ErrDuplicateID {
logrus.Debugf("error creating image: %q", err)
return errors.Wrapf(err, "error creating image %q", intendedID)
return errors.Wrapf(err, "creating image %q", intendedID)
}
img, err = s.imageRef.transport.store.Image(intendedID)
if err != nil {
return errors.Wrapf(err, "error reading image %q", intendedID)
return errors.Wrapf(err, "reading image %q", intendedID)
}
if img.TopLayer != lastLayer {
logrus.Debugf("error creating image: image with ID %q exists, but uses different layers", intendedID)
@ -1011,6 +1102,19 @@ func (s *storageImageDestination) Commit(ctx context.Context, unparsedToplevel t
} else {
logrus.Debugf("created new image ID %q", img.ID)
}
// Clean up the unfinished image on any error.
// (Is this the right thing to do if the image has existed before?)
commitSucceeded := false
defer func() {
if !commitSucceeded {
logrus.Errorf("Updating image %q (old names %v) failed, deleting it", img.ID, oldNames)
if _, err := s.imageRef.transport.store.DeleteImage(img.ID, true); err != nil {
logrus.Errorf("Error deleting incomplete image %q: %v", img.ID, err)
}
}
}()
// Add the non-layer blobs as data items. Since we only share layers, they should all be in files, so
// we just need to screen out the ones that are actually layers to get the list of non-layers.
dataBlobs := make(map[digest.Digest]struct{})
@ -1023,90 +1127,62 @@ func (s *storageImageDestination) Commit(ctx context.Context, unparsedToplevel t
for blob := range dataBlobs {
v, err := ioutil.ReadFile(s.filenames[blob])
if err != nil {
return errors.Wrapf(err, "error copying non-layer blob %q to image", blob)
return errors.Wrapf(err, "copying non-layer blob %q to image", blob)
}
if err := s.imageRef.transport.store.SetImageBigData(img.ID, blob.String(), v, manifest.Digest); err != nil {
if _, err2 := s.imageRef.transport.store.DeleteImage(img.ID, true); err2 != nil {
logrus.Debugf("error deleting incomplete image %q: %v", img.ID, err2)
}
logrus.Debugf("error saving big data %q for image %q: %v", blob.String(), img.ID, err)
return errors.Wrapf(err, "error saving big data %q for image %q", blob.String(), img.ID)
return errors.Wrapf(err, "saving big data %q for image %q", blob.String(), img.ID)
}
}
// Save the unparsedToplevel's manifest.
if len(toplevelManifest) != 0 {
// Save the unparsedToplevel's manifest if it differs from the per-platform one, which is saved below.
if len(toplevelManifest) != 0 && !bytes.Equal(toplevelManifest, s.manifest) {
manifestDigest, err := manifest.Digest(toplevelManifest)
if err != nil {
return errors.Wrapf(err, "error digesting top-level manifest")
return errors.Wrapf(err, "digesting top-level manifest")
}
key := manifestBigDataKey(manifestDigest)
if err := s.imageRef.transport.store.SetImageBigData(img.ID, key, toplevelManifest, manifest.Digest); err != nil {
if _, err2 := s.imageRef.transport.store.DeleteImage(img.ID, true); err2 != nil {
logrus.Debugf("error deleting incomplete image %q: %v", img.ID, err2)
}
logrus.Debugf("error saving top-level manifest for image %q: %v", img.ID, err)
return errors.Wrapf(err, "error saving top-level manifest for image %q", img.ID)
return errors.Wrapf(err, "saving top-level manifest for image %q", img.ID)
}
}
// Save the image's manifest. Allow looking it up by digest by using the key convention defined by the Store.
// Record the manifest twice: using a digest-specific key to allow references to that specific digest instance,
// and using storage.ImageDigestBigDataKey for future users that don’t specify any digest and for compatibility with older readers.
manifestDigest, err := manifest.Digest(s.manifest)
if err != nil {
return errors.Wrapf(err, "error computing manifest digest")
}
key := manifestBigDataKey(manifestDigest)
key := manifestBigDataKey(s.manifestDigest)
if err := s.imageRef.transport.store.SetImageBigData(img.ID, key, s.manifest, manifest.Digest); err != nil {
if _, err2 := s.imageRef.transport.store.DeleteImage(img.ID, true); err2 != nil {
logrus.Debugf("error deleting incomplete image %q: %v", img.ID, err2)
}
logrus.Debugf("error saving manifest for image %q: %v", img.ID, err)
return errors.Wrapf(err, "error saving manifest for image %q", img.ID)
return errors.Wrapf(err, "saving manifest for image %q", img.ID)
}
key = storage.ImageDigestBigDataKey
if err := s.imageRef.transport.store.SetImageBigData(img.ID, key, s.manifest, manifest.Digest); err != nil {
if _, err2 := s.imageRef.transport.store.DeleteImage(img.ID, true); err2 != nil {
logrus.Debugf("error deleting incomplete image %q: %v", img.ID, err2)
}
logrus.Debugf("error saving manifest for image %q: %v", img.ID, err)
return errors.Wrapf(err, "error saving manifest for image %q", img.ID)
return errors.Wrapf(err, "saving manifest for image %q", img.ID)
}
// Save the signatures, if we have any.
if len(s.signatures) > 0 {
if err := s.imageRef.transport.store.SetImageBigData(img.ID, "signatures", s.signatures, manifest.Digest); err != nil {
if _, err2 := s.imageRef.transport.store.DeleteImage(img.ID, true); err2 != nil {
logrus.Debugf("error deleting incomplete image %q: %v", img.ID, err2)
}
logrus.Debugf("error saving signatures for image %q: %v", img.ID, err)
return errors.Wrapf(err, "error saving signatures for image %q", img.ID)
return errors.Wrapf(err, "saving signatures for image %q", img.ID)
}
}
for instanceDigest, signatures := range s.signatureses {
key := signatureBigDataKey(instanceDigest)
if err := s.imageRef.transport.store.SetImageBigData(img.ID, key, signatures, manifest.Digest); err != nil {
if _, err2 := s.imageRef.transport.store.DeleteImage(img.ID, true); err2 != nil {
logrus.Debugf("error deleting incomplete image %q: %v", img.ID, err2)
}
logrus.Debugf("error saving signatures for image %q: %v", img.ID, err)
return errors.Wrapf(err, "error saving signatures for image %q", img.ID)
return errors.Wrapf(err, "saving signatures for image %q", img.ID)
}
}
// Save our metadata.
metadata, err := json.Marshal(s)
if err != nil {
if _, err2 := s.imageRef.transport.store.DeleteImage(img.ID, true); err2 != nil {
logrus.Debugf("error deleting incomplete image %q: %v", img.ID, err2)
}
logrus.Debugf("error encoding metadata for image %q: %v", img.ID, err)
return errors.Wrapf(err, "error encoding metadata for image %q", img.ID)
return errors.Wrapf(err, "encoding metadata for image %q", img.ID)
}
if len(metadata) != 0 {
if err = s.imageRef.transport.store.SetMetadata(img.ID, string(metadata)); err != nil {
if _, err2 := s.imageRef.transport.store.DeleteImage(img.ID, true); err2 != nil {
logrus.Debugf("error deleting incomplete image %q: %v", img.ID, err2)
}
logrus.Debugf("error saving metadata for image %q: %v", img.ID, err)
return errors.Wrapf(err, "error saving metadata for image %q", img.ID)
return errors.Wrapf(err, "saving metadata for image %q", img.ID)
}
logrus.Debugf("saved image metadata %q", string(metadata))
}
@ -1121,14 +1197,13 @@ func (s *storageImageDestination) Commit(ctx context.Context, unparsedToplevel t
names = append(names, oldNames...)
}
if err := s.imageRef.transport.store.SetNames(img.ID, names); err != nil {
if _, err2 := s.imageRef.transport.store.DeleteImage(img.ID, true); err2 != nil {
logrus.Debugf("error deleting incomplete image %q: %v", img.ID, err2)
}
logrus.Debugf("error setting names %v on image %q: %v", names, img.ID, err)
return errors.Wrapf(err, "error setting names %v on image %q", names, img.ID)
return errors.Wrapf(err, "setting names %v on image %q", names, img.ID)
}
logrus.Debugf("set names of image %q to %v", img.ID, names)
}
commitSucceeded = true
return nil
}
@ -1145,9 +1220,14 @@ func (s *storageImageDestination) SupportedManifestMIMETypes() []string {
// PutManifest writes the manifest to the destination.
func (s *storageImageDestination) PutManifest(ctx context.Context, manifestBlob []byte, instanceDigest *digest.Digest) error {
digest, err := manifest.Digest(manifestBlob)
if err != nil {
return err
}
newBlob := make([]byte, len(manifestBlob))
copy(newBlob, manifestBlob)
s.manifest = newBlob
s.manifestDigest = digest
return nil
}
@ -1189,13 +1269,10 @@ func (s *storageImageDestination) PutSignatures(ctx context.Context, signatures
if instanceDigest == nil {
s.signatures = sigblob
s.SignatureSizes = sizes
}
if instanceDigest == nil && len(s.manifest) > 0 {
manifestDigest, err := manifest.Digest(s.manifest)
if err != nil {
return err
if len(s.manifest) > 0 {
manifestDigest := s.manifestDigest
instanceDigest = &manifestDigest
}
instanceDigest = &manifestDigest
}
if instanceDigest != nil {
s.signatureses[*instanceDigest] = sigblob
@ -1211,12 +1288,12 @@ func (s *storageImageSource) getSize() (int64, error) {
// Size up the data blobs.
dataNames, err := s.imageRef.transport.store.ListImageBigData(s.image.ID)
if err != nil {
return -1, errors.Wrapf(err, "error reading image %q", s.image.ID)
return -1, errors.Wrapf(err, "reading image %q", s.image.ID)
}
for _, dataName := range dataNames {
bigSize, err := s.imageRef.transport.store.ImageBigDataSize(s.image.ID, dataName)
if err != nil {
return -1, errors.Wrapf(err, "error reading data blob size %q for %q", dataName, s.image.ID)
return -1, errors.Wrapf(err, "reading data blob size %q for %q", dataName, s.image.ID)
}
sum += bigSize
}

91
vendor/github.com/containers/image/v5/storage/storage_reference.go generated vendored
View File

@ -11,7 +11,6 @@ import (
"github.com/containers/image/v5/types"
"github.com/containers/storage"
digest "github.com/opencontainers/go-digest"
imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
)
@ -62,18 +61,17 @@ func imageMatchesRepo(image *storage.Image, ref reference.Named) bool {
return false
}
// imageMatchesSystemContext checks if the passed-in image both contains a
// manifest that matches the passed-in digest, and identifies itself as being
// appropriate for running on the system that matches sys.
// If we somehow ended up sharing the same storage among multiple types of
// systems, and managed to download multiple images from the same manifest
// list, their image records will all contain copies of the manifest list, and
// this check will help us decide which of them we want to return when we've
// been asked to resolve an image reference that uses the list's digest to a
// specific image ID.
func imageMatchesSystemContext(store storage.Store, img *storage.Image, manifestDigest digest.Digest, sys *types.SystemContext) bool {
// First, check if the image record has a manifest that matches the
// specified digest.
// multiArchImageMatchesSystemContext returns true if if the passed-in image both contains a
// multi-arch manifest that matches the passed-in digest, and the image is the per-platform
// image instance that matches sys.
//
// See the comment in storageReference.ResolveImage explaining why
// this check is necessary.
func multiArchImageMatchesSystemContext(store storage.Store, img *storage.Image, manifestDigest digest.Digest, sys *types.SystemContext) bool {
// Load the manifest that matches the specified digest.
// We don't need to care about storage.ImageDigestBigDataKey because
// manifests lists are only stored into storage by c/image versions
// that know about manifestBigDataKey, and only using that key.
key := manifestBigDataKey(manifestDigest)
manifestBytes, err := store.ImageBigData(img.ID, key)
if err != nil {
@ -83,56 +81,22 @@ func imageMatchesSystemContext(store storage.Store, img *storage.Image, manifest
// the digest of the instance that matches the current system, and try
// to load that manifest from the image record, and use it.
manifestType := manifest.GuessMIMEType(manifestBytes)
if manifest.MIMETypeIsMultiImage(manifestType) {
list, err := manifest.ListFromBlob(manifestBytes, manifestType)
if err != nil {
return false
}
manifestDigest, err = list.ChooseInstance(sys)
if err != nil {
return false
}
key = manifestBigDataKey(manifestDigest)
manifestBytes, err = store.ImageBigData(img.ID, key)
if err != nil {
return false
}
manifestType = manifest.GuessMIMEType(manifestBytes)
if !manifest.MIMETypeIsMultiImage(manifestType) {
// manifestDigest directly specifies a per-platform image, so we aren't
// choosing among different variants.
return false
}
// Load the image's configuration blob.
m, err := manifest.FromBlob(manifestBytes, manifestType)
list, err := manifest.ListFromBlob(manifestBytes, manifestType)
if err != nil {
return false
}
getConfig := func(blobInfo types.BlobInfo) ([]byte, error) {
return store.ImageBigData(img.ID, blobInfo.Digest.String())
}
ii, err := m.Inspect(getConfig)
chosenInstance, err := list.ChooseInstance(sys)
if err != nil {
return false
}
// Build a dummy index containing one instance and information about
// the image's target system from the image's configuration.
index := manifest.OCI1IndexFromComponents([]imgspecv1.Descriptor{{
MediaType: imgspecv1.MediaTypeImageManifest,
Digest: manifestDigest,
Size: int64(len(manifestBytes)),
Platform: &imgspecv1.Platform{
OS: ii.Os,
Architecture: ii.Architecture,
},
}}, nil)
// Check that ChooseInstance() would select this image for this system,
// from a list of images.
instanceDigest, err := index.ChooseInstance(sys)
if err != nil {
return false
}
// Double-check that we can read the runnable image's manifest from the
// image record.
key = manifestBigDataKey(instanceDigest)
key = manifestBigDataKey(chosenInstance)
_, err = store.ImageBigData(img.ID, key)
return err == nil
return err == nil // true if img.ID is based on chosenInstance.
}
// Resolve the reference's name to an image ID in the store, if there's already
@ -152,11 +116,24 @@ func (s *storageReference) resolveImage(sys *types.SystemContext) (*storage.Imag
// Look for an image with the specified digest that has the same name,
// though possibly with a different tag or digest, as a Name value, so
// that the canonical reference can be implicitly resolved to the image.
//
// Typically there should be at most one such image, because the same
// manifest digest implies the same config, and we choose the storage ID
// based on the config (deduplicating images), except:
// - the user can explicitly specify an ID when creating the image.
// In this case we don't have a preference among the alternatives.
// - when pulling an image from a multi-platform manifest list, we also
// store the manifest list in the image; this allows referencing a
// per-platform image using the manifest list digest, but that also
// means that we can have multiple genuinely different images in the
// storage matching the same manifest list digest (if pulled using different
// SystemContext.{OS,Architecture,Variant}Choice to the same storage).
// In this case we prefer the image matching the current SystemContext.
images, err := s.transport.store.ImagesByDigest(digested.Digest())
if err == nil && len(images) > 0 {
for _, image := range images {
if imageMatchesRepo(image, s.named) {
if loadedImage == nil || imageMatchesSystemContext(s.transport.store, image, digested.Digest(), sys) {
if loadedImage == nil || multiArchImageMatchesSystemContext(s.transport.store, image, digested.Digest(), sys) {
loadedImage = image
s.id = image.ID
}
@ -172,7 +149,7 @@ func (s *storageReference) resolveImage(sys *types.SystemContext) (*storage.Imag
if loadedImage == nil {
img, err := s.transport.store.Image(s.id)
if err != nil {
return nil, errors.Wrapf(err, "error reading image %q", s.id)
return nil, errors.Wrapf(err, "reading image %q", s.id)
}
loadedImage = img
}

4
vendor/github.com/containers/image/v5/storage/storage_transport.go generated vendored
View File

@ -172,7 +172,7 @@ func (s storageTransport) ParseStoreReference(store storage.Store, ref string) (
var err error
named, err = reference.ParseNormalizedNamed(ref)
if err != nil {
return nil, errors.Wrapf(err, "error parsing named reference %q", ref)
return nil, errors.Wrapf(err, "parsing named reference %q", ref)
}
named = reference.TagNameOnly(named)
}
@ -303,7 +303,7 @@ func (s storageTransport) GetStoreImage(store storage.Store, ref types.ImageRefe
}
if sref, ok := ref.(*storageReference); ok {
tmpRef := *sref
if img, err := tmpRef.resolveImage(&types.SystemContext{}); err == nil {
if img, err := tmpRef.resolveImage(nil); err == nil {
return img, nil
}
}

7
vendor/github.com/containers/image/v5/types/types.go generated vendored
View File

@ -334,6 +334,9 @@ type ImageDestination interface {
// MUST be called after PutManifest (signatures may reference manifest contents).
PutSignatures(ctx context.Context, signatures [][]byte, instanceDigest *digest.Digest) error
// Commit marks the process of storing the image as successful and asks for the image to be persisted.
// unparsedToplevel contains data about the top-level manifest of the source (which may be a single-arch image or a manifest list
// if PutManifest was only called for the single-arch image with instanceDigest == nil), primarily to allow lookups by the
// original manifest list digest, if desired.
// WARNING: This does not have any transactional semantics:
// - Uploaded data MAY be visible to others before Commit() is called
// - Uploaded data MAY be removed or MAY remain around if Close() is called without Commit() (i.e. rollback is allowed but not guaranteed)
@ -595,12 +598,12 @@ type SystemContext struct {
// === docker.Transport overrides ===
// If not "", a directory containing a CA certificate (ending with ".crt"),
// a client certificate (ending with ".cert") and a client certificate key
// (ending with ".key") used when talking to a Docker Registry.
// (ending with ".key") used when talking to a container registry.
DockerCertPath string
// If not "", overrides the system’s default path for a directory containing host[:port] subdirectories with the same structure as DockerCertPath above.
// Ignored if DockerCertPath is non-empty.
DockerPerHostCertDirPath string
// Allow contacting docker registries over HTTP, or HTTPS with failed TLS verification. Note that this does not affect other TLS connections.
// Allow contacting container registries over HTTP, or HTTPS with failed TLS verification. Note that this does not affect other TLS connections.
DockerInsecureSkipTLSVerify OptionalBool
// if nil, the library tries to parse ~/.docker/config.json to retrieve credentials
// Ignored if DockerBearerRegistryToken is non-empty.

4
vendor/github.com/containers/image/v5/version/version.go generated vendored
View File

@ -6,9 +6,9 @@ const (
// VersionMajor is for an API incompatible changes
VersionMajor = 5
// VersionMinor is for functionality in a backwards-compatible manner
VersionMinor = 13
VersionMinor = 14
// VersionPatch is for backwards-compatible bug fixes
VersionPatch = 2
VersionPatch = 0
// VersionDev indicates development branch. Releases will be empty string.
VersionDev = ""

513
vendor/github.com/containers/storage/pkg/chunked/compression.go generated vendored Normal file
View File

@ -0,0 +1,513 @@
package chunked
import (
"bytes"
"encoding/base64"
"encoding/binary"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"time"
"github.com/containers/storage/pkg/ioutils"
"github.com/klauspost/compress/zstd"
digest "github.com/opencontainers/go-digest"
"github.com/pkg/errors"
"github.com/vbatts/tar-split/archive/tar"
)
type zstdTOC struct {
Version int `json:"version"`
Entries []zstdFileMetadata `json:"entries"`
}
type zstdFileMetadata struct {
Type string `json:"type"`
Name string `json:"name"`
Linkname string `json:"linkName,omitempty"`
Mode int64 `json:"mode,omitempty"`
Size int64 `json:"size"`
UID int `json:"uid"`
GID int `json:"gid"`
ModTime time.Time `json:"modtime"`
AccessTime time.Time `json:"accesstime"`
ChangeTime time.Time `json:"changetime"`
Devmajor int64 `json:"devMajor"`
Devminor int64 `json:"devMinor"`
Xattrs map[string]string `json:"xattrs,omitempty"`
Digest string `json:"digest,omitempty"`
Offset int64 `json:"offset,omitempty"`
EndOffset int64 `json:"endOffset,omitempty"`
// Currently chunking is not supported.
ChunkSize int64 `json:"chunkSize,omitempty"`
ChunkOffset int64 `json:"chunkOffset,omitempty"`
ChunkDigest string `json:"chunkDigest,omitempty"`
}
const (
TypeReg = "reg"
TypeChunk = "chunk"
TypeLink = "hardlink"
TypeChar = "char"
TypeBlock = "block"
TypeDir = "dir"
TypeFifo = "fifo"
TypeSymlink = "symlink"
)
var tarTypes = map[byte]string{
tar.TypeReg: TypeReg,
tar.TypeRegA: TypeReg,
tar.TypeLink: TypeLink,
tar.TypeChar: TypeChar,
tar.TypeBlock: TypeBlock,
tar.TypeDir: TypeDir,
tar.TypeFifo: TypeFifo,
tar.TypeSymlink: TypeSymlink,
}
var typesToTar = map[string]byte{
TypeReg: tar.TypeReg,
TypeLink: tar.TypeLink,
TypeChar: tar.TypeChar,
TypeBlock: tar.TypeBlock,
TypeDir: tar.TypeDir,
TypeFifo: tar.TypeFifo,
TypeSymlink: tar.TypeSymlink,
}
func getType(t byte) (string, error) {
r, found := tarTypes[t]
if !found {
return "", fmt.Errorf("unknown tarball type: %v", t)
}
return r, nil
}
func typeToTarType(t string) (byte, error) {
r, found := typesToTar[t]
if !found {
return 0, fmt.Errorf("unknown type: %v", t)
}
return r, nil
}
const (
manifestChecksumKey = "io.containers.zstd-chunked.manifest-checksum"
manifestInfoKey = "io.containers.zstd-chunked.manifest-position"
// manifestTypeCRFS is a manifest file compatible with the CRFS TOC file.
manifestTypeCRFS = 1
// footerSizeSupported is the footer size supported by this implementation.
// Newer versions of the image format might increase this value, so reject
// any version that is not supported.
footerSizeSupported = 40
)
var (
// when the zstd decoder encounters a skippable frame + 1 byte for the size, it
// will ignore it.
// https://tools.ietf.org/html/rfc8478#section-3.1.2
skippableFrameMagic = []byte{0x50, 0x2a, 0x4d, 0x18}
zstdChunkedFrameMagic = []byte{0x47, 0x6e, 0x55, 0x6c, 0x49, 0x6e, 0x55, 0x78}
)
func isZstdChunkedFrameMagic(data []byte) bool {
if len(data) < 8 {
return false
}
return bytes.Equal(zstdChunkedFrameMagic, data[:8])
}
// readZstdChunkedManifest reads the zstd:chunked manifest from the seekable stream blobStream. The blob total size must
// be specified.
// This function uses the io.containers.zstd-chunked. annotations when specified.
func readZstdChunkedManifest(blobStream ImageSourceSeekable, blobSize int64, annotations map[string]string) ([]byte, error) {
footerSize := int64(footerSizeSupported)
if blobSize <= footerSize {
return nil, errors.New("blob too small")
}
manifestChecksumAnnotation := annotations[manifestChecksumKey]
if manifestChecksumAnnotation == "" {
return nil, fmt.Errorf("manifest checksum annotation %q not found", manifestChecksumKey)
}
var offset, length, lengthUncompressed, manifestType uint64
if offsetMetadata := annotations[manifestInfoKey]; offsetMetadata != "" {
if _, err := fmt.Sscanf(offsetMetadata, "%d:%d:%d:%d", &offset, &length, &lengthUncompressed, &manifestType); err != nil {
return nil, err
}
} else {
chunk := ImageSourceChunk{
Offset: uint64(blobSize - footerSize),
Length: uint64(footerSize),
}
parts, errs, err := blobStream.GetBlobAt([]ImageSourceChunk{chunk})
if err != nil {
return nil, err
}
var reader io.ReadCloser
select {
case r := <-parts:
reader = r
case err := <-errs:
return nil, err
}
footer := make([]byte, footerSize)
if _, err := io.ReadFull(reader, footer); err != nil {
return nil, err
}
offset = binary.LittleEndian.Uint64(footer[0:8])
length = binary.LittleEndian.Uint64(footer[8:16])
lengthUncompressed = binary.LittleEndian.Uint64(footer[16:24])
manifestType = binary.LittleEndian.Uint64(footer[24:32])
if !isZstdChunkedFrameMagic(footer[32:40]) {
return nil, errors.New("invalid magic number")
}
}
if manifestType != manifestTypeCRFS {
return nil, errors.New("invalid manifest type")
}
// set a reasonable limit
if length > (1<<20)*50 {
return nil, errors.New("manifest too big")
}
if lengthUncompressed > (1<<20)*50 {
return nil, errors.New("manifest too big")
}
chunk := ImageSourceChunk{
Offset: offset,
Length: length,
}
parts, errs, err := blobStream.GetBlobAt([]ImageSourceChunk{chunk})
if err != nil {
return nil, err
}
var reader io.ReadCloser
select {
case r := <-parts:
reader = r
case err := <-errs:
return nil, err
}
manifest := make([]byte, length)
if _, err := io.ReadFull(reader, manifest); err != nil {
return nil, err
}
manifestDigester := digest.Canonical.Digester()
manifestChecksum := manifestDigester.Hash()
if _, err := manifestChecksum.Write(manifest); err != nil {
return nil, err
}
d, err := digest.Parse(manifestChecksumAnnotation)
if err != nil {
return nil, err
}
if manifestDigester.Digest() != d {
return nil, errors.New("invalid manifest checksum")
}
decoder, err := zstd.NewReader(nil)
if err != nil {
return nil, err
}
defer decoder.Close()
b := make([]byte, 0, lengthUncompressed)
if decoded, err := decoder.DecodeAll(manifest, b); err == nil {
return decoded, nil
}
return manifest, nil
}
func appendZstdSkippableFrame(dest io.Writer, data []byte) error {
if _, err := dest.Write(skippableFrameMagic); err != nil {
return err
}
var size []byte = make([]byte, 4)
binary.LittleEndian.PutUint32(size, uint32(len(data)))
if _, err := dest.Write(size); err != nil {
return err
}
if _, err := dest.Write(data); err != nil {
return err
}
return nil
}
func writeZstdChunkedManifest(dest io.Writer, outMetadata map[string]string, offset uint64, metadata []zstdFileMetadata, level int) error {
// 8 is the size of the zstd skippable frame header + the frame size
manifestOffset := offset + 8
toc := zstdTOC{
Version: 1,
Entries: metadata,
}
// Generate the manifest
manifest, err := json.Marshal(toc)
if err != nil {
return err
}
var compressedBuffer bytes.Buffer
zstdWriter, err := zstdWriterWithLevel(&compressedBuffer, level)
if err != nil {
return err
}
if _, err := zstdWriter.Write(manifest); err != nil {
zstdWriter.Close()
return err
}
if err := zstdWriter.Close(); err != nil {
return err
}
compressedManifest := compressedBuffer.Bytes()
manifestDigester := digest.Canonical.Digester()
manifestChecksum := manifestDigester.Hash()
if _, err := manifestChecksum.Write(compressedManifest); err != nil {
return err
}
outMetadata[manifestChecksumKey] = manifestDigester.Digest().String()
outMetadata[manifestInfoKey] = fmt.Sprintf("%d:%d:%d:%d", manifestOffset, len(compressedManifest), len(manifest), manifestTypeCRFS)
if err := appendZstdSkippableFrame(dest, compressedManifest); err != nil {
return err
}
// Store the offset to the manifest and its size in LE order
var manifestDataLE []byte = make([]byte, footerSizeSupported)
binary.LittleEndian.PutUint64(manifestDataLE, manifestOffset)
binary.LittleEndian.PutUint64(manifestDataLE[8:], uint64(len(compressedManifest)))
binary.LittleEndian.PutUint64(manifestDataLE[16:], uint64(len(manifest)))
binary.LittleEndian.PutUint64(manifestDataLE[24:], uint64(manifestTypeCRFS))
copy(manifestDataLE[32:], zstdChunkedFrameMagic)
return appendZstdSkippableFrame(dest, manifestDataLE)
}
func writeZstdChunkedStream(destFile io.Writer, outMetadata map[string]string, reader io.Reader, level int) error {
// total written so far. Used to retrieve partial offsets in the file
dest := ioutils.NewWriteCounter(destFile)
tr := tar.NewReader(reader)
tr.RawAccounting = true
buf := make([]byte, 4096)
zstdWriter, err := zstdWriterWithLevel(dest, level)
if err != nil {
return err
}
defer func() {
if zstdWriter != nil {
zstdWriter.Close()
zstdWriter.Flush()
}
}()
restartCompression := func() (int64, error) {
var offset int64
if zstdWriter != nil {
if err := zstdWriter.Close(); err != nil {
return 0, err
}
if err := zstdWriter.Flush(); err != nil {
return 0, err
}
offset = dest.Count
zstdWriter.Reset(dest)
}
return offset, nil
}
var metadata []zstdFileMetadata
for {
hdr, err := tr.Next()
if err != nil {
if err == io.EOF {
break
}
return err
}
rawBytes := tr.RawBytes()
if _, err := zstdWriter.Write(rawBytes); err != nil {
return err
}
payloadDigester := digest.Canonical.Digester()
payloadChecksum := payloadDigester.Hash()
payloadDest := io.MultiWriter(payloadChecksum, zstdWriter)
// Now handle the payload, if any
var startOffset, endOffset int64
checksum := ""
for {
read, errRead := tr.Read(buf)
if errRead != nil && errRead != io.EOF {
return err
}
// restart the compression only if there is
// a payload.
if read > 0 {
if startOffset == 0 {
startOffset, err = restartCompression()
if err != nil {
return err
}
}
_, err := payloadDest.Write(buf[:read])
if err != nil {
return err
}
}
if errRead == io.EOF {
if startOffset > 0 {
endOffset, err = restartCompression()
if err != nil {
return err
}
checksum = payloadDigester.Digest().String()
}
break
}
}
typ, err := getType(hdr.Typeflag)
if err != nil {
return err
}
xattrs := make(map[string]string)
for k, v := range hdr.Xattrs {
xattrs[k] = base64.StdEncoding.EncodeToString([]byte(v))
}
m := zstdFileMetadata{
Type: typ,
Name: hdr.Name,
Linkname: hdr.Linkname,
Mode: hdr.Mode,
Size: hdr.Size,
UID: hdr.Uid,
GID: hdr.Gid,
ModTime: hdr.ModTime,
AccessTime: hdr.AccessTime,
ChangeTime: hdr.ChangeTime,
Devmajor: hdr.Devmajor,
Devminor: hdr.Devminor,
Xattrs: xattrs,
Digest: checksum,
Offset: startOffset,
EndOffset: endOffset,
// ChunkSize is 0 for the last chunk
ChunkSize: 0,
ChunkOffset: 0,
ChunkDigest: checksum,
}
metadata = append(metadata, m)
}
rawBytes := tr.RawBytes()
if _, err := zstdWriter.Write(rawBytes); err != nil {
return err
}
if err := zstdWriter.Flush(); err != nil {
return err
}
if err := zstdWriter.Close(); err != nil {
return err
}
zstdWriter = nil
return writeZstdChunkedManifest(dest, outMetadata, uint64(dest.Count), metadata, level)
}
type zstdChunkedWriter struct {
tarSplitOut *io.PipeWriter
tarSplitErr chan error
}
func (w zstdChunkedWriter) Close() error {
err := <-w.tarSplitErr
if err != nil {
w.tarSplitOut.Close()
return err
}
return w.tarSplitOut.Close()
}
func (w zstdChunkedWriter) Write(p []byte) (int, error) {
select {
case err := <-w.tarSplitErr:
w.tarSplitOut.Close()
return 0, err
default:
return w.tarSplitOut.Write(p)
}
}
// zstdChunkedWriterWithLevel writes a zstd compressed tarball where each file is
// compressed separately so it can be addressed separately. Idea based on CRFS:
// https://github.com/google/crfs
// The difference with CRFS is that the zstd compression is used instead of gzip.
// The reason for it is that zstd supports embedding metadata ignored by the decoder
// as part of the compressed stream.
// A manifest json file with all the metadata is appended at the end of the tarball
// stream, using zstd skippable frames.
// The final file will look like:
// [FILE_1][FILE_2]..[FILE_N][SKIPPABLE FRAME 1][SKIPPABLE FRAME 2]
// Where:
// [FILE_N]: [ZSTD HEADER][TAR HEADER][PAYLOAD FILE_N][ZSTD FOOTER]
// [SKIPPABLE FRAME 1]: [ZSTD SKIPPABLE FRAME, SIZE=MANIFEST LENGTH][MANIFEST]
// [SKIPPABLE FRAME 2]: [ZSTD SKIPPABLE FRAME, SIZE=16][MANIFEST_OFFSET][MANIFEST_LENGTH][MANIFEST_LENGTH_UNCOMPRESSED][MANIFEST_TYPE][CHUNKED_ZSTD_MAGIC_NUMBER]
// MANIFEST_OFFSET, MANIFEST_LENGTH, MANIFEST_LENGTH_UNCOMPRESSED and CHUNKED_ZSTD_MAGIC_NUMBER are 64 bits unsigned in little endian format.
func zstdChunkedWriterWithLevel(out io.Writer, metadata map[string]string, level int) (io.WriteCloser, error) {
ch := make(chan error, 1)
r, w := io.Pipe()
go func() {
ch <- writeZstdChunkedStream(out, metadata, r, level)
io.Copy(ioutil.Discard, r)
r.Close()
close(ch)
}()
return zstdChunkedWriter{
tarSplitOut: w,
tarSplitErr: ch,
}, nil
}
func zstdWriterWithLevel(dest io.Writer, level int) (*zstd.Encoder, error) {
el := zstd.EncoderLevelFromZstd(level)
return zstd.NewWriter(dest, zstd.WithEncoderLevel(el))
}
// ZstdCompressor is a CompressorFunc for the zstd compression algorithm.
func ZstdCompressor(r io.Writer, metadata map[string]string, level *int) (io.WriteCloser, error) {
if level == nil {
l := 3
level = &l
}
return zstdChunkedWriterWithLevel(r, metadata, *level)
}

26
vendor/github.com/containers/storage/pkg/chunked/storage.go generated vendored Normal file
View File

@ -0,0 +1,26 @@
package chunked
import (
"fmt"
"io"
)
// ImageSourceChunk is a portion of a blob.
type ImageSourceChunk struct {
Offset uint64
Length uint64
}
// ImageSourceSeekable is an image source that permits to fetch chunks of the entire blob.
type ImageSourceSeekable interface {
// GetBlobAt returns a stream for the specified blob.
GetBlobAt([]ImageSourceChunk) (chan io.ReadCloser, chan error, error)
}
// ErrBadRequest is returned when the request is not valid
type ErrBadRequest struct {
}
func (e ErrBadRequest) Error() string {
return fmt.Sprintf("bad request")
}

874
vendor/github.com/containers/storage/pkg/chunked/storage_linux.go generated vendored Normal file
View File

@ -0,0 +1,874 @@
package chunked
import (
archivetar "archive/tar"
"context"
"encoding/base64"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"sort"
"strings"
"syscall"
"time"
storage "github.com/containers/storage"
graphdriver "github.com/containers/storage/drivers"
driversCopy "github.com/containers/storage/drivers/copy"
"github.com/containers/storage/pkg/archive"
"github.com/containers/storage/pkg/idtools"
"github.com/containers/storage/types"
"github.com/klauspost/compress/zstd"
digest "github.com/opencontainers/go-digest"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"github.com/vbatts/tar-split/archive/tar"
"golang.org/x/sys/unix"
)
const (
maxNumberMissingChunks = 1024
newFileFlags = (unix.O_CREAT | unix.O_TRUNC | unix.O_WRONLY | unix.O_EXCL)
containersOverrideXattr = "user.containers.override_stat"
bigDataKey = "zstd-chunked-manifest"
)
type chunkedZstdDiffer struct {
stream ImageSourceSeekable
manifest []byte
layersMetadata map[string][]zstdFileMetadata
layersTarget map[string]string
}
func timeToTimespec(time time.Time) (ts unix.Timespec) {
if time.IsZero() {
// Return UTIME_OMIT special value
ts.Sec = 0
ts.Nsec = ((1 << 30) - 2)
return
}
return unix.NsecToTimespec(time.UnixNano())
}
func copyFileContent(src, destFile, root string, dirfd int, missingDirsMode, mode os.FileMode) (*os.File, int64, error) {
st, err := os.Stat(src)
if err != nil {
return nil, -1, err
}
copyWithFileRange, copyWithFileClone := true, true
// If the destination file already exists, we shouldn't blow it away
dstFile, err := openFileUnderRoot(destFile, root, dirfd, newFileFlags, mode)
if err != nil {
return nil, -1, err
}
err = driversCopy.CopyRegularToFile(src, dstFile, st, &copyWithFileRange, &copyWithFileClone)
if err != nil {
dstFile.Close()
return nil, -1, err
}
return dstFile, st.Size(), err
}
func prepareOtherLayersCache(layersMetadata map[string][]zstdFileMetadata) map[string]map[string]*zstdFileMetadata {
maps := make(map[string]map[string]*zstdFileMetadata)
for layerID, v := range layersMetadata {
r := make(map[string]*zstdFileMetadata)
for i := range v {
r[v[i].Digest] = &v[i]
}
maps[layerID] = r
}
return maps
}
func getLayersCache(store storage.Store) (map[string][]zstdFileMetadata, map[string]string, error) {
allLayers, err := store.Layers()
if err != nil {
return nil, nil, err
}
layersMetadata := make(map[string][]zstdFileMetadata)
layersTarget := make(map[string]string)
for _, r := range allLayers {
manifestReader, err := store.LayerBigData(r.ID, bigDataKey)
if err != nil {
continue
}
defer manifestReader.Close()
manifest, err := ioutil.ReadAll(manifestReader)
if err != nil {
return nil, nil, err
}
var toc zstdTOC
if err := json.Unmarshal(manifest, &toc); err != nil {
continue
}
layersMetadata[r.ID] = toc.Entries
target, err := store.DifferTarget(r.ID)
if err != nil {
return nil, nil, err
}
layersTarget[r.ID] = target
}
return layersMetadata, layersTarget, nil
}
// GetDiffer returns a differ than can be used with ApplyDiffWithDiffer.
func GetDiffer(ctx context.Context, store storage.Store, blobSize int64, annotations map[string]string, iss ImageSourceSeekable) (graphdriver.Differ, error) {
if _, ok := annotations[manifestChecksumKey]; ok {
return makeZstdChunkedDiffer(ctx, store, blobSize, annotations, iss)
}
return nil, errors.New("blob type not supported for partial retrieval")
}
func makeZstdChunkedDiffer(ctx context.Context, store storage.Store, blobSize int64, annotations map[string]string, iss ImageSourceSeekable) (*chunkedZstdDiffer, error) {
manifest, err := readZstdChunkedManifest(iss, blobSize, annotations)
if err != nil {
return nil, err
}
layersMetadata, layersTarget, err := getLayersCache(store)
if err != nil {
return nil, err
}
return &chunkedZstdDiffer{
stream: iss,
manifest: manifest,
layersMetadata: layersMetadata,
layersTarget: layersTarget,
}, nil
}
func findFileInOtherLayers(file zstdFileMetadata, root string, dirfd int, layersMetadata map[string]map[string]*zstdFileMetadata, layersTarget map[string]string, missingDirsMode os.FileMode) (*os.File, int64, error) {
// this is ugly, needs to be indexed
for layerID, checksums := range layersMetadata {
m, found := checksums[file.Digest]
if !found {
continue
}
source, ok := layersTarget[layerID]
if !ok {
continue
}
srcDirfd, err := unix.Open(source, unix.O_RDONLY, 0)
if err != nil {
continue
}
defer unix.Close(srcDirfd)
srcFile, err := openFileUnderRoot(m.Name, source, srcDirfd, unix.O_RDONLY, 0)
if err != nil {
continue
}
defer srcFile.Close()
srcPath := fmt.Sprintf("/proc/self/fd/%d", srcFile.Fd())
dstFile, written, err := copyFileContent(srcPath, file.Name, root, dirfd, missingDirsMode, 0)
if err != nil {
continue
}
return dstFile, written, nil
}
return nil, 0, nil
}
func getFileDigest(f *os.File) (digest.Digest, error) {
digester := digest.Canonical.Digester()
if _, err := io.Copy(digester.Hash(), f); err != nil {
return "", err
}
return digester.Digest(), nil
}
// findFileOnTheHost checks whether the requested file already exist on the host and copies the file content from there if possible.
// It is currently implemented to look only at the file with the same path. Ideally it can detect the same content also at different
// paths.
func findFileOnTheHost(file zstdFileMetadata, root string, dirfd int, missingDirsMode os.FileMode) (*os.File, int64, error) {
sourceFile := filepath.Clean(filepath.Join("/", file.Name))
if !strings.HasPrefix(sourceFile, "/usr/") {
// limit host deduplication to files under /usr.
return nil, 0, nil
}
st, err := os.Stat(sourceFile)
if err != nil || !st.Mode().IsRegular() {
return nil, 0, nil
}
if st.Size() != file.Size {
return nil, 0, nil
}
fd, err := unix.Open(sourceFile, unix.O_RDONLY|unix.O_NONBLOCK, 0)
if err != nil {
return nil, 0, nil
}
f := os.NewFile(uintptr(fd), "fd")
defer f.Close()
manifestChecksum, err := digest.Parse(file.Digest)
if err != nil {
return nil, 0, err
}
checksum, err := getFileDigest(f)
if err != nil {
return nil, 0, err
}
if checksum != manifestChecksum {
return nil, 0, nil
}
dstFile, written, err := copyFileContent(fmt.Sprintf("/proc/self/fd/%d", fd), file.Name, root, dirfd, missingDirsMode, 0)
if err != nil {
return nil, 0, nil
}
// calculate the checksum again to make sure the file wasn't modified while it was copied
if _, err := f.Seek(0, 0); err != nil {
return nil, 0, err
}
checksum, err = getFileDigest(f)
if err != nil {
return nil, 0, err
}
if checksum != manifestChecksum {
return nil, 0, nil
}
return dstFile, written, nil
}
func maybeDoIDRemap(manifest []zstdFileMetadata, options *archive.TarOptions) error {
if options.ChownOpts == nil && len(options.UIDMaps) == 0 || len(options.GIDMaps) == 0 {
return nil
}
idMappings := idtools.NewIDMappingsFromMaps(options.UIDMaps, options.GIDMaps)
for i := range manifest {
if options.ChownOpts != nil {
manifest[i].UID = options.ChownOpts.UID
manifest[i].GID = options.ChownOpts.GID
} else {
pair := idtools.IDPair{
UID: manifest[i].UID,
GID: manifest[i].GID,
}
var err error
manifest[i].UID, manifest[i].GID, err = idMappings.ToContainer(pair)
if err != nil {
return err
}
}
}
return nil
}
type missingFile struct {
File *zstdFileMetadata
Gap int64
}
func (m missingFile) Length() int64 {
return m.File.EndOffset - m.File.Offset
}
type missingChunk struct {
RawChunk ImageSourceChunk
Files []missingFile
}
func setFileAttrs(file *os.File, mode os.FileMode, metadata *zstdFileMetadata, options *archive.TarOptions) error {
if file == nil || file.Fd() < 0 {
return errors.Errorf("invalid file")
}
fd := int(file.Fd())
t, err := typeToTarType(metadata.Type)
if err != nil {
return err
}
if t == tar.TypeSymlink {
return nil
}
if err := unix.Fchown(fd, metadata.UID, metadata.GID); err != nil {
if !options.IgnoreChownErrors {
return err
}
}
for k, v := range metadata.Xattrs {
data, err := base64.StdEncoding.DecodeString(v)
if err != nil {
return err
}
if err := unix.Fsetxattr(fd, k, data, 0); err != nil {
return err
}
}
ts := []unix.Timespec{timeToTimespec(metadata.AccessTime), timeToTimespec(metadata.ModTime)}
if err := unix.UtimesNanoAt(fd, "", ts, 0); err != nil && errors.Is(err, unix.ENOSYS) {
return err
}
if err := unix.Fchmod(fd, uint32(mode)); err != nil {
return err
}
return nil
}
func openFileUnderRoot(name, root string, dirfd int, flags uint64, mode os.FileMode) (*os.File, error) {
how := unix.OpenHow{
Flags: flags,
Mode: uint64(mode & 07777),
Resolve: unix.RESOLVE_IN_ROOT,
}
fd, err := unix.Openat2(dirfd, name, &how)
if err != nil {
return nil, err
}
return os.NewFile(uintptr(fd), name), nil
}
func createFileFromZstdStream(dest string, dirfd int, reader io.Reader, missingDirsMode, mode os.FileMode, metadata *zstdFileMetadata, options *archive.TarOptions) (err error) {
file, err := openFileUnderRoot(metadata.Name, dest, dirfd, newFileFlags, 0)
if err != nil {
return err
}
defer func() {
err2 := file.Close()
if err == nil {
err = err2
}
}()
z, err := zstd.NewReader(reader)
if err != nil {
return err
}
defer z.Close()
digester := digest.Canonical.Digester()
checksum := digester.Hash()
_, err = z.WriteTo(io.MultiWriter(file, checksum))
if err != nil {
return err
}
manifestChecksum, err := digest.Parse(metadata.Digest)
if err != nil {
return err
}
if digester.Digest() != manifestChecksum {
return fmt.Errorf("checksum mismatch for %q", dest)
}
return setFileAttrs(file, mode, metadata, options)
}
func storeMissingFiles(streams chan io.ReadCloser, errs chan error, dest string, dirfd int, missingChunks []missingChunk, missingDirsMode os.FileMode, options *archive.TarOptions) error {
for mc := 0; ; mc++ {
var part io.ReadCloser
select {
case p := <-streams:
part = p
case err := <-errs:
return err
}
if part == nil {
if mc == len(missingChunks) {
break
}
return errors.Errorf("invalid stream returned %d %d", mc, len(missingChunks))
}
if mc == len(missingChunks) {
return errors.Errorf("too many chunks returned")
}
for _, mf := range missingChunks[mc].Files {
if mf.Gap > 0 {
limitReader := io.LimitReader(part, mf.Gap)
_, err := io.Copy(ioutil.Discard, limitReader)
if err != nil {
return err
}
continue
}
limitReader := io.LimitReader(part, mf.Length())
if err := createFileFromZstdStream(dest, dirfd, limitReader, missingDirsMode, os.FileMode(mf.File.Mode), mf.File, options); err != nil {
part.Close()
return err
}
}
part.Close()
}
return nil
}
func mergeMissingChunks(missingChunks []missingChunk, target int) []missingChunk {
if len(missingChunks) <= target {
return missingChunks
}
getGap := func(missingChunks []missingChunk, i int) int {
prev := missingChunks[i-1].RawChunk.Offset + missingChunks[i-1].RawChunk.Length
return int(missingChunks[i].RawChunk.Offset - prev)
}
// this implementation doesn't account for duplicates, so it could merge
// more than necessary to reach the specified target. Since target itself
// is a heuristic value, it doesn't matter.
var gaps []int
for i := 1; i < len(missingChunks); i++ {
gaps = append(gaps, getGap(missingChunks, i))
}
sort.Ints(gaps)
toShrink := len(missingChunks) - target
targetValue := gaps[toShrink-1]
newMissingChunks := missingChunks[0:1]
for i := 1; i < len(missingChunks); i++ {
gap := getGap(missingChunks, i)
if gap > targetValue {
newMissingChunks = append(newMissingChunks, missingChunks[i])
} else {
prev := &newMissingChunks[len(newMissingChunks)-1]
gapFile := missingFile{
Gap: int64(gap),
}
prev.RawChunk.Length += uint64(gap) + missingChunks[i].RawChunk.Length
prev.Files = append(append(prev.Files, gapFile), missingChunks[i].Files...)
}
}
return newMissingChunks
}
func retrieveMissingFiles(input *chunkedZstdDiffer, dest string, dirfd int, missingChunks []missingChunk, missingDirsMode os.FileMode, options *archive.TarOptions) error {
var chunksToRequest []ImageSourceChunk
for _, c := range missingChunks {
chunksToRequest = append(chunksToRequest, c.RawChunk)
}
// There are some missing files. Prepare a multirange request for the missing chunks.
var streams chan io.ReadCloser
var err error
var errs chan error
for {
streams, errs, err = input.stream.GetBlobAt(chunksToRequest)
if err == nil {
break
}
if _, ok := err.(ErrBadRequest); ok {
requested := len(missingChunks)
// If the server cannot handle at least 64 chunks in a single request, just give up.
if requested < 64 {
return err
}
// Merge more chunks to request
missingChunks = mergeMissingChunks(missingChunks, requested/2)
continue
}
return err
}
if err := storeMissingFiles(streams, errs, dest, dirfd, missingChunks, missingDirsMode, options); err != nil {
return err
}
return nil
}
func safeMkdir(target string, dirfd int, mode os.FileMode, metadata *zstdFileMetadata, options *archive.TarOptions) error {
parent := filepath.Dir(metadata.Name)
base := filepath.Base(metadata.Name)
parentFd := dirfd
if parent != "." {
parentFile, err := openFileUnderRoot(parent, target, dirfd, unix.O_DIRECTORY|unix.O_PATH|unix.O_RDONLY, 0)
if err != nil {
return err
}
defer parentFile.Close()
parentFd = int(parentFile.Fd())
}
if err := unix.Mkdirat(parentFd, base, uint32(mode)); err != nil {
if !os.IsExist(err) {
return err
}
}
file, err := openFileUnderRoot(metadata.Name, target, dirfd, unix.O_RDONLY, 0)
if err != nil {
return err
}
defer file.Close()
return setFileAttrs(file, mode, metadata, options)
}
func safeLink(target string, dirfd int, mode os.FileMode, metadata *zstdFileMetadata, options *archive.TarOptions) error {
sourceFile, err := openFileUnderRoot(metadata.Linkname, target, dirfd, unix.O_RDONLY, 0)
if err != nil {
return err
}
defer sourceFile.Close()
destDir, destBase := filepath.Dir(metadata.Name), filepath.Base(metadata.Name)
destDirFd := dirfd
if destDir != "." {
f, err := openFileUnderRoot(destDir, target, dirfd, unix.O_RDONLY, 0)
if err != nil {
return err
}
defer f.Close()
destDirFd = int(f.Fd())
}
err = unix.Linkat(int(sourceFile.Fd()), "", destDirFd, destBase, unix.AT_EMPTY_PATH)
if err != nil {
return err
}
newFile, err := openFileUnderRoot(metadata.Name, target, dirfd, unix.O_WRONLY, 0)
if err != nil {
return err
}
defer newFile.Close()
return setFileAttrs(newFile, mode, metadata, options)
}
func safeSymlink(target string, dirfd int, mode os.FileMode, metadata *zstdFileMetadata, options *archive.TarOptions) error {
destDir, destBase := filepath.Dir(metadata.Name), filepath.Base(metadata.Name)
destDirFd := dirfd
if destDir != "." {
f, err := openFileUnderRoot(destDir, target, dirfd, unix.O_RDONLY, 0)
if err != nil {
return err
}
defer f.Close()
destDirFd = int(f.Fd())
}
return unix.Symlinkat(metadata.Linkname, destDirFd, destBase)
}
type whiteoutHandler struct {
Dirfd int
Root string
}
func (d whiteoutHandler) Setxattr(path, name string, value []byte) error {
file, err := openFileUnderRoot(path, d.Root, d.Dirfd, unix.O_RDONLY, 0)
if err != nil {
return err
}
defer file.Close()
return unix.Fsetxattr(int(file.Fd()), name, value, 0)
}
func (d whiteoutHandler) Mknod(path string, mode uint32, dev int) error {
dir := filepath.Dir(path)
base := filepath.Base(path)
dirfd := d.Dirfd
if dir != "" {
dir, err := openFileUnderRoot(dir, d.Root, d.Dirfd, unix.O_RDONLY, 0)
if err != nil {
return err
}
defer dir.Close()
dirfd = int(dir.Fd())
}
return unix.Mknodat(dirfd, base, mode, dev)
}
func checkChownErr(err error, name string, uid, gid int) error {
if errors.Is(err, syscall.EINVAL) {
return errors.Wrapf(err, "potentially insufficient UIDs or GIDs available in user namespace (requested %d:%d for %s): Check /etc/subuid and /etc/subgid", uid, gid, name)
}
return err
}
func (d whiteoutHandler) Chown(path string, uid, gid int) error {
file, err := openFileUnderRoot(path, d.Root, d.Dirfd, unix.O_PATH, 0)
if err != nil {
return err
}
defer file.Close()
if err := unix.Fchownat(int(file.Fd()), "", uid, gid, unix.AT_EMPTY_PATH); err != nil {
var stat unix.Stat_t
if unix.Fstat(int(file.Fd()), &stat) == nil {
if stat.Uid == uint32(uid) && stat.Gid == uint32(gid) {
return nil
}
}
return checkChownErr(err, path, uid, gid)
}
return nil
}
type hardLinkToCreate struct {
dest string
dirfd int
mode os.FileMode
metadata *zstdFileMetadata
}
func (d *chunkedZstdDiffer) ApplyDiff(dest string, options *archive.TarOptions) (graphdriver.DriverWithDifferOutput, error) {
bigData := map[string][]byte{
bigDataKey: d.manifest,
}
output := graphdriver.DriverWithDifferOutput{
Differ: d,
BigData: bigData,
}
storeOpts, err := types.DefaultStoreOptionsAutoDetectUID()
if err != nil {
return output, err
}
enableHostDedup := false
if value := storeOpts.PullOptions["enable_host_deduplication"]; strings.ToLower(value) == "true" {
enableHostDedup = true
}
// Generate the manifest
var toc zstdTOC
if err := json.Unmarshal(d.manifest, &toc); err != nil {
return output, err
}
whiteoutConverter := archive.GetWhiteoutConverter(options.WhiteoutFormat, options.WhiteoutData)
var missingChunks []missingChunk
var mergedEntries []zstdFileMetadata
if err := maybeDoIDRemap(toc.Entries, options); err != nil {
return output, err
}
for _, e := range toc.Entries {
if e.Type == TypeChunk {
l := len(mergedEntries)
if l == 0 || mergedEntries[l-1].Type != TypeReg {
return output, errors.New("chunk type without a regular file")
}
mergedEntries[l-1].EndOffset = e.EndOffset
continue
}
mergedEntries = append(mergedEntries, e)
}
if options.ForceMask != nil {
uid, gid, mode, err := archive.GetFileOwner(dest)
if err == nil {
value := fmt.Sprintf("%d:%d:0%o", uid, gid, mode)
if err := unix.Setxattr(dest, containersOverrideXattr, []byte(value), 0); err != nil {
return output, err
}
}
}
dirfd, err := unix.Open(dest, unix.O_RDONLY|unix.O_PATH, 0)
if err != nil {
return output, err
}
defer unix.Close(dirfd)
otherLayersCache := prepareOtherLayersCache(d.layersMetadata)
missingDirsMode := os.FileMode(0700)
if options.ForceMask != nil {
missingDirsMode = *options.ForceMask
}
// hardlinks can point to missing files. So create them after all files
// are retrieved
var hardLinks []hardLinkToCreate
missingChunksSize, totalChunksSize := int64(0), int64(0)
for i, r := range mergedEntries {
if options.ForceMask != nil {
value := fmt.Sprintf("%d:%d:0%o", r.UID, r.GID, r.Mode&07777)
r.Xattrs[containersOverrideXattr] = base64.StdEncoding.EncodeToString([]byte(value))
r.Mode = int64(*options.ForceMask)
}
mode := os.FileMode(r.Mode)
r.Name = filepath.Clean(r.Name)
r.Linkname = filepath.Clean(r.Linkname)
t, err := typeToTarType(r.Type)
if err != nil {
return output, err
}
if whiteoutConverter != nil {
hdr := archivetar.Header{
Typeflag: t,
Name: r.Name,
Linkname: r.Linkname,
Size: r.Size,
Mode: r.Mode,
Uid: r.UID,
Gid: r.GID,
}
handler := whiteoutHandler{
Dirfd: dirfd,
Root: dest,
}
writeFile, err := whiteoutConverter.ConvertReadWithHandler(&hdr, r.Name, &handler)
if err != nil {
return output, err
}
if !writeFile {
continue
}
}
switch t {
case tar.TypeReg:
// Create directly empty files.
if r.Size == 0 {
// Used to have a scope for cleanup.
createEmptyFile := func() error {
file, err := openFileUnderRoot(r.Name, dest, dirfd, newFileFlags, 0)
if err != nil {
return err
}
defer file.Close()
if err := setFileAttrs(file, mode, &r, options); err != nil {
return err
}
return nil
}
if err := createEmptyFile(); err != nil {
return output, err
}
continue
}
case tar.TypeDir:
if err := safeMkdir(dest, dirfd, mode, &r, options); err != nil {
return output, err
}
continue
case tar.TypeLink:
dest := dest
dirfd := dirfd
mode := mode
r := r
hardLinks = append(hardLinks, hardLinkToCreate{
dest: dest,
dirfd: dirfd,
mode: mode,
metadata: &r,
})
continue
case tar.TypeSymlink:
if err := safeSymlink(dest, dirfd, mode, &r, options); err != nil {
return output, err
}
continue
case tar.TypeChar:
case tar.TypeBlock:
case tar.TypeFifo:
/* Ignore. */
default:
return output, fmt.Errorf("invalid type %q", t)
}
totalChunksSize += r.Size
dstFile, _, err := findFileInOtherLayers(r, dest, dirfd, otherLayersCache, d.layersTarget, missingDirsMode)
if err != nil {
return output, err
}
if dstFile != nil {
if err := setFileAttrs(dstFile, mode, &r, options); err != nil {
dstFile.Close()
return output, err
}
dstFile.Close()
continue
}
if enableHostDedup {
dstFile, _, err = findFileOnTheHost(r, dest, dirfd, missingDirsMode)
if err != nil {
return output, err
}
if dstFile != nil {
if err := setFileAttrs(dstFile, mode, &r, options); err != nil {
dstFile.Close()
return output, err
}
dstFile.Close()
continue
}
}
missingChunksSize += r.Size
if t == tar.TypeReg {
rawChunk := ImageSourceChunk{
Offset: uint64(r.Offset),
Length: uint64(r.EndOffset - r.Offset),
}
file := missingFile{
File: &toc.Entries[i],
}
missingChunks = append(missingChunks, missingChunk{
RawChunk: rawChunk,
Files: []missingFile{
file,
},
})
}
}
// There are some missing files. Prepare a multirange request for the missing chunks.
if len(missingChunks) > 0 {
missingChunks = mergeMissingChunks(missingChunks, maxNumberMissingChunks)
if err := retrieveMissingFiles(d, dest, dirfd, missingChunks, missingDirsMode, options); err != nil {
return output, err
}
}
for _, m := range hardLinks {
if err := safeLink(m.dest, m.dirfd, m.mode, m.metadata, options); err != nil {
return output, err
}
}
if totalChunksSize > 0 {
logrus.Debugf("Missing %d bytes out of %d (%.2f %%)", missingChunksSize, totalChunksSize, float32(missingChunksSize*100.0)/float32(totalChunksSize))
}
return output, nil
}

16
vendor/github.com/containers/storage/pkg/chunked/storage_unsupported.go generated vendored Normal file
View File

@ -0,0 +1,16 @@
// +build !linux
package chunked
import (
"context"
storage "github.com/containers/storage"
graphdriver "github.com/containers/storage/drivers"
"github.com/pkg/errors"
)
// GetDiffer returns a differ than can be used with ApplyDiffWithDiffer.
func GetDiffer(ctx context.Context, store storage.Store, blobSize int64, annotations map[string]string, iss ImageSourceSeekable) (graphdriver.Differ, error) {
return nil, errors.New("format not supported on this architecture")
}

7
vendor/github.com/vbauerster/mpb/v7/cwriter/util_zos.go generated vendored Normal file
View File

@ -0,0 +1,7 @@
// +build zos
package cwriter
import "golang.org/x/sys/unix"
const ioctlReadTermios = unix.TCGETS

2
vendor/github.com/vbauerster/mpb/v7/go.mod generated vendored
View File

@ -4,7 +4,7 @@ require (
github.com/VividCortex/ewma v1.2.0
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d
github.com/mattn/go-runewidth v0.0.13
golang.org/x/sys v0.0.0-20210603125802-9665404d3644
golang.org/x/sys v0.0.0-20210616094352-59db8d763f22
)
go 1.14

4
vendor/github.com/vbauerster/mpb/v7/go.sum generated vendored
View File

@ -6,5 +6,5 @@ github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4
github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
golang.org/x/sys v0.0.0-20210603125802-9665404d3644 h1:CA1DEQ4NdKphKeL70tvsWNdT5oFh1lOjihRcEDROi0I=
golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210616094352-59db8d763f22 h1:RqytpXGR1iVNX7psjB3ff8y7sNFinVFvkx1c8SjBkio=
golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

1
vendor/golang.org/x/sys/unix/mkerrors.sh generated vendored
View File

@ -563,6 +563,7 @@ ccflags="$@"
$2 ~ /^KEYCTL_/ ||
$2 ~ /^PERF_/ ||
$2 ~ /^SECCOMP_MODE_/ ||
$2 ~ /^SEEK_/ ||
$2 ~ /^SPLICE_/ ||
$2 ~ /^SYNC_FILE_RANGE_/ ||
$2 !~ /^AUDIT_RECORD_MAGIC/ &&

33
vendor/golang.org/x/sys/unix/syscall_darwin.go generated vendored
View File

@ -13,6 +13,7 @@
package unix
import (
"fmt"
"runtime"
"syscall"
"unsafe"
@ -398,6 +399,38 @@ func GetsockoptXucred(fd, level, opt int) (*Xucred, error) {
return x, err
}
func SysctlKinfoProcSlice(name string) ([]KinfoProc, error) {
mib, err := sysctlmib(name)
if err != nil {
return nil, err
}
// Find size.
n := uintptr(0)
if err := sysctl(mib, nil, &n, nil, 0); err != nil {
return nil, err
}
if n == 0 {
return nil, nil
}
if n%SizeofKinfoProc != 0 {
return nil, fmt.Errorf("sysctl() returned a size of %d, which is not a multiple of %d", n, SizeofKinfoProc)
}
// Read into buffer of that size.
buf := make([]KinfoProc, n/SizeofKinfoProc)
if err := sysctl(mib, (*byte)(unsafe.Pointer(&buf[0])), &n, nil, 0); err != nil {
return nil, err
}
if n%SizeofKinfoProc != 0 {
return nil, fmt.Errorf("sysctl() returned a size of %d, which is not a multiple of %d", n, SizeofKinfoProc)
}
// The actual call may return less than the original reported required
// size so ensure we deal with that.
return buf[:n/SizeofKinfoProc], nil
}
//sys sendfile(infd int, outfd int, offset int64, len *int64, hdtr unsafe.Pointer, flags int) (err error)
/*

5
vendor/golang.org/x/sys/unix/zerrors_darwin_amd64.go generated vendored
View File

@ -1262,6 +1262,11 @@ const (
SCM_RIGHTS = 0x1
SCM_TIMESTAMP = 0x2
SCM_TIMESTAMP_MONOTONIC = 0x4
SEEK_CUR = 0x1
SEEK_DATA = 0x4
SEEK_END = 0x2
SEEK_HOLE = 0x3
SEEK_SET = 0x0
SHUT_RD = 0x0
SHUT_RDWR = 0x2
SHUT_WR = 0x1

5
vendor/golang.org/x/sys/unix/zerrors_darwin_arm64.go generated vendored
View File

@ -1262,6 +1262,11 @@ const (
SCM_RIGHTS = 0x1
SCM_TIMESTAMP = 0x2
SCM_TIMESTAMP_MONOTONIC = 0x4
SEEK_CUR = 0x1
SEEK_DATA = 0x4
SEEK_END = 0x2
SEEK_HOLE = 0x3
SEEK_SET = 0x0
SHUT_RD = 0x0
SHUT_RDWR = 0x2
SHUT_WR = 0x1

5
vendor/golang.org/x/sys/unix/zerrors_freebsd_386.go generated vendored
View File

@ -1297,6 +1297,11 @@ const (
SCM_RIGHTS = 0x1
SCM_TIMESTAMP = 0x2
SCM_TIME_INFO = 0x7
SEEK_CUR = 0x1
SEEK_DATA = 0x3
SEEK_END = 0x2
SEEK_HOLE = 0x4
SEEK_SET = 0x0
SHUT_RD = 0x0
SHUT_RDWR = 0x2
SHUT_WR = 0x1

5
vendor/golang.org/x/sys/unix/zerrors_freebsd_amd64.go generated vendored
View File

@ -1298,6 +1298,11 @@ const (
SCM_RIGHTS = 0x1
SCM_TIMESTAMP = 0x2
SCM_TIME_INFO = 0x7
SEEK_CUR = 0x1
SEEK_DATA = 0x3
SEEK_END = 0x2
SEEK_HOLE = 0x4
SEEK_SET = 0x0
SHUT_RD = 0x0
SHUT_RDWR = 0x2
SHUT_WR = 0x1

5
vendor/golang.org/x/sys/unix/zerrors_freebsd_arm.go generated vendored
View File

@ -1276,6 +1276,11 @@ const (
SCM_CREDS = 0x3
SCM_RIGHTS = 0x1
SCM_TIMESTAMP = 0x2
SEEK_CUR = 0x1
SEEK_DATA = 0x3
SEEK_END = 0x2
SEEK_HOLE = 0x4
SEEK_SET = 0x0
SHUT_RD = 0x0
SHUT_RDWR = 0x2
SHUT_WR = 0x1

5
vendor/golang.org/x/sys/unix/zerrors_freebsd_arm64.go generated vendored
View File

@ -1298,6 +1298,11 @@ const (
SCM_RIGHTS = 0x1
SCM_TIMESTAMP = 0x2
SCM_TIME_INFO = 0x7
SEEK_CUR = 0x1
SEEK_DATA = 0x3
SEEK_END = 0x2
SEEK_HOLE = 0x4
SEEK_SET = 0x0
SHUT_RD = 0x0
SHUT_RDWR = 0x2
SHUT_WR = 0x1

6
vendor/golang.org/x/sys/unix/zerrors_linux.go generated vendored
View File

@ -2284,6 +2284,12 @@ const (
SECCOMP_MODE_FILTER = 0x2
SECCOMP_MODE_STRICT = 0x1
SECURITYFS_MAGIC = 0x73636673
SEEK_CUR = 0x1
SEEK_DATA = 0x3
SEEK_END = 0x2
SEEK_HOLE = 0x4
SEEK_MAX = 0x4
SEEK_SET = 0x0
SELINUX_MAGIC = 0xf97cff8c
SHUT_RD = 0x0
SHUT_RDWR = 0x2

104
vendor/golang.org/x/sys/unix/ztypes_darwin_amd64.go generated vendored
View File

@ -535,3 +535,107 @@ type CtlInfo struct {
Id uint32
Name [96]byte
}
const SizeofKinfoProc = 0x288
type Eproc struct {
Paddr uintptr
Sess uintptr
Pcred Pcred
Ucred Ucred
Vm Vmspace
Ppid int32
Pgid int32
Jobc int16
Tdev int32
Tpgid int32
Tsess uintptr
Wmesg [8]int8
Xsize int32
Xrssize int16
Xccount int16
Xswrss int16
Flag int32
Login [12]int8
Spare [4]int32
_ [4]byte
}
type ExternProc struct {
P_starttime Timeval
P_vmspace *Vmspace
P_sigacts uintptr
P_flag int32
P_stat int8
P_pid int32
P_oppid int32
P_dupfd int32
User_stack *int8
Exit_thread *byte
P_debugger int32
Sigwait int32
P_estcpu uint32
P_cpticks int32
P_pctcpu uint32
P_wchan *byte
P_wmesg *int8
P_swtime uint32
P_slptime uint32
P_realtimer Itimerval
P_rtime Timeval
P_uticks uint64
P_sticks uint64
P_iticks uint64
P_traceflag int32
P_tracep uintptr
P_siglist int32
P_textvp uintptr
P_holdcnt int32
P_sigmask uint32
P_sigignore uint32
P_sigcatch uint32
P_priority uint8
P_usrpri uint8
P_nice int8
P_comm [17]int8
P_pgrp uintptr
P_addr uintptr
P_xstat uint16
P_acflag uint16
P_ru *Rusage
}
type Itimerval struct {
Interval Timeval
Value Timeval
}
type KinfoProc struct {
Proc ExternProc
Eproc Eproc
}
type Vmspace struct {
Dummy int32
Dummy2 *int8
Dummy3 [5]int32
Dummy4 [3]*int8
}
type Pcred struct {
Pc_lock [72]int8
Pc_ucred uintptr
P_ruid uint32
P_svuid uint32
P_rgid uint32
P_svgid uint32
P_refcnt int32
_ [4]byte
}
type Ucred struct {
Ref int32
Uid uint32
Ngroups int16
Groups [16]uint32
}

104
vendor/golang.org/x/sys/unix/ztypes_darwin_arm64.go generated vendored
View File

@ -535,3 +535,107 @@ type CtlInfo struct {
Id uint32
Name [96]byte
}
const SizeofKinfoProc = 0x288
type Eproc struct {
Paddr uintptr
Sess uintptr
Pcred Pcred
Ucred Ucred
Vm Vmspace
Ppid int32
Pgid int32
Jobc int16
Tdev int32
Tpgid int32
Tsess uintptr
Wmesg [8]int8
Xsize int32
Xrssize int16
Xccount int16
Xswrss int16
Flag int32
Login [12]int8
Spare [4]int32
_ [4]byte
}
type ExternProc struct {
P_starttime Timeval
P_vmspace *Vmspace
P_sigacts uintptr
P_flag int32
P_stat int8
P_pid int32
P_oppid int32
P_dupfd int32
User_stack *int8
Exit_thread *byte
P_debugger int32
Sigwait int32
P_estcpu uint32
P_cpticks int32
P_pctcpu uint32
P_wchan *byte
P_wmesg *int8
P_swtime uint32
P_slptime uint32
P_realtimer Itimerval
P_rtime Timeval
P_uticks uint64
P_sticks uint64
P_iticks uint64
P_traceflag int32
P_tracep uintptr
P_siglist int32
P_textvp uintptr
P_holdcnt int32
P_sigmask uint32
P_sigignore uint32
P_sigcatch uint32
P_priority uint8
P_usrpri uint8
P_nice int8
P_comm [17]int8
P_pgrp uintptr
P_addr uintptr
P_xstat uint16
P_acflag uint16
P_ru *Rusage
}
type Itimerval struct {
Interval Timeval
Value Timeval
}
type KinfoProc struct {
Proc ExternProc
Eproc Eproc
}
type Vmspace struct {
Dummy int32
Dummy2 *int8
Dummy3 [5]int32
Dummy4 [3]*int8
}
type Pcred struct {
Pc_lock [72]int8
Pc_ucred uintptr
P_ruid uint32
P_svuid uint32
P_rgid uint32
P_svgid uint32
P_refcnt int32
_ [4]byte
}
type Ucred struct {
Ref int32
Uid uint32
Ngroups int16
Groups [16]uint32
}

3
vendor/golang.org/x/sys/unix/ztypes_dragonfly_amd64.go generated vendored
View File

@ -431,6 +431,9 @@ type Winsize struct {
const (
AT_FDCWD = 0xfffafdcd
AT_SYMLINK_NOFOLLOW = 0x1
AT_REMOVEDIR = 0x2
AT_EACCESS = 0x4
AT_SYMLINK_FOLLOW = 0x8
)
type PollFd struct {

5
vendor/golang.org/x/sys/unix/ztypes_freebsd_386.go generated vendored
View File

@ -672,9 +672,10 @@ type Winsize struct {
const (
AT_FDCWD = -0x64
AT_REMOVEDIR = 0x800
AT_SYMLINK_FOLLOW = 0x400
AT_EACCESS = 0x100
AT_SYMLINK_NOFOLLOW = 0x200
AT_SYMLINK_FOLLOW = 0x400
AT_REMOVEDIR = 0x800
)
type PollFd struct {

5
vendor/golang.org/x/sys/unix/ztypes_freebsd_amd64.go generated vendored
View File

@ -675,9 +675,10 @@ type Winsize struct {
const (
AT_FDCWD = -0x64
AT_REMOVEDIR = 0x800
AT_SYMLINK_FOLLOW = 0x400
AT_EACCESS = 0x100
AT_SYMLINK_NOFOLLOW = 0x200
AT_SYMLINK_FOLLOW = 0x400
AT_REMOVEDIR = 0x800
)
type PollFd struct {

5
vendor/golang.org/x/sys/unix/ztypes_freebsd_arm.go generated vendored
View File

@ -656,9 +656,10 @@ type Winsize struct {
const (
AT_FDCWD = -0x64
AT_REMOVEDIR = 0x800
AT_SYMLINK_FOLLOW = 0x400
AT_EACCESS = 0x100
AT_SYMLINK_NOFOLLOW = 0x200
AT_SYMLINK_FOLLOW = 0x400
AT_REMOVEDIR = 0x800
)
type PollFd struct {

5
vendor/golang.org/x/sys/unix/ztypes_freebsd_arm64.go generated vendored
View File

@ -653,9 +653,10 @@ type Winsize struct {
const (
AT_FDCWD = -0x64
AT_REMOVEDIR = 0x800
AT_SYMLINK_FOLLOW = 0x400
AT_EACCESS = 0x100
AT_SYMLINK_NOFOLLOW = 0x200
AT_SYMLINK_FOLLOW = 0x400
AT_REMOVEDIR = 0x800
)
type PollFd struct {

4
vendor/golang.org/x/sys/unix/ztypes_netbsd_386.go generated vendored
View File

@ -445,8 +445,10 @@ type Ptmget struct {
const (
AT_FDCWD = -0x64
AT_SYMLINK_FOLLOW = 0x400
AT_EACCESS = 0x100
AT_SYMLINK_NOFOLLOW = 0x200
AT_SYMLINK_FOLLOW = 0x400
AT_REMOVEDIR = 0x800
)
type PollFd struct {

4
vendor/golang.org/x/sys/unix/ztypes_netbsd_amd64.go generated vendored
View File

@ -453,8 +453,10 @@ type Ptmget struct {
const (
AT_FDCWD = -0x64
AT_SYMLINK_FOLLOW = 0x400
AT_EACCESS = 0x100
AT_SYMLINK_NOFOLLOW = 0x200
AT_SYMLINK_FOLLOW = 0x400
AT_REMOVEDIR = 0x800
)
type PollFd struct {

4
vendor/golang.org/x/sys/unix/ztypes_netbsd_arm.go generated vendored
View File

@ -450,8 +450,10 @@ type Ptmget struct {
const (
AT_FDCWD = -0x64
AT_SYMLINK_FOLLOW = 0x400
AT_EACCESS = 0x100
AT_SYMLINK_NOFOLLOW = 0x200
AT_SYMLINK_FOLLOW = 0x400
AT_REMOVEDIR = 0x800
)
type PollFd struct {

4
vendor/golang.org/x/sys/unix/ztypes_netbsd_arm64.go generated vendored
View File

@ -453,8 +453,10 @@ type Ptmget struct {
const (
AT_FDCWD = -0x64
AT_SYMLINK_FOLLOW = 0x400
AT_EACCESS = 0x100
AT_SYMLINK_NOFOLLOW = 0x200
AT_SYMLINK_FOLLOW = 0x400
AT_REMOVEDIR = 0x800
)
type PollFd struct {

4
vendor/golang.org/x/sys/unix/ztypes_openbsd_386.go generated vendored
View File

@ -438,8 +438,10 @@ type Winsize struct {
const (
AT_FDCWD = -0x64
AT_SYMLINK_FOLLOW = 0x4
AT_EACCESS = 0x1
AT_SYMLINK_NOFOLLOW = 0x2
AT_SYMLINK_FOLLOW = 0x4
AT_REMOVEDIR = 0x8
)
type PollFd struct {

4
vendor/golang.org/x/sys/unix/ztypes_openbsd_amd64.go generated vendored
View File

@ -438,8 +438,10 @@ type Winsize struct {
const (
AT_FDCWD = -0x64
AT_SYMLINK_FOLLOW = 0x4
AT_EACCESS = 0x1
AT_SYMLINK_NOFOLLOW = 0x2
AT_SYMLINK_FOLLOW = 0x4
AT_REMOVEDIR = 0x8
)
type PollFd struct {

4
vendor/golang.org/x/sys/unix/ztypes_openbsd_arm.go generated vendored
View File

@ -439,8 +439,10 @@ type Winsize struct {
const (
AT_FDCWD = -0x64
AT_SYMLINK_FOLLOW = 0x4
AT_EACCESS = 0x1
AT_SYMLINK_NOFOLLOW = 0x2
AT_SYMLINK_FOLLOW = 0x4
AT_REMOVEDIR = 0x8
)
type PollFd struct {

4
vendor/golang.org/x/sys/unix/ztypes_openbsd_arm64.go generated vendored
View File

@ -432,8 +432,10 @@ type Winsize struct {
const (
AT_FDCWD = -0x64
AT_SYMLINK_FOLLOW = 0x4
AT_EACCESS = 0x1
AT_SYMLINK_NOFOLLOW = 0x2
AT_SYMLINK_FOLLOW = 0x4
AT_REMOVEDIR = 0x8
)
type PollFd struct {

4
vendor/golang.org/x/sys/unix/ztypes_openbsd_mips64.go generated vendored
View File

@ -432,8 +432,10 @@ type Winsize struct {
const (
AT_FDCWD = -0x64
AT_SYMLINK_FOLLOW = 0x4
AT_EACCESS = 0x1
AT_SYMLINK_NOFOLLOW = 0x2
AT_SYMLINK_FOLLOW = 0x4
AT_REMOVEDIR = 0x8
)
type PollFd struct {

7
vendor/modules.txt vendored
View File

@ -125,7 +125,7 @@ github.com/containers/common/pkg/umask
github.com/containers/common/version
# github.com/containers/conmon v2.0.20+incompatible
github.com/containers/conmon/runner/config
# github.com/containers/image/v5 v5.13.2
# github.com/containers/image/v5 v5.14.0
github.com/containers/image/v5/copy
github.com/containers/image/v5/directory
github.com/containers/image/v5/directory/explicitfilepath
@ -213,6 +213,7 @@ github.com/containers/storage/drivers/windows
github.com/containers/storage/drivers/zfs
github.com/containers/storage/pkg/archive
github.com/containers/storage/pkg/chrootarchive
github.com/containers/storage/pkg/chunked
github.com/containers/storage/pkg/config
github.com/containers/storage/pkg/devicemapper
github.com/containers/storage/pkg/directory
@ -607,7 +608,7 @@ github.com/vbauerster/mpb/v6
github.com/vbauerster/mpb/v6/cwriter
github.com/vbauerster/mpb/v6/decor
github.com/vbauerster/mpb/v6/internal
# github.com/vbauerster/mpb/v7 v7.0.2
# github.com/vbauerster/mpb/v7 v7.0.3
github.com/vbauerster/mpb/v7
github.com/vbauerster/mpb/v7/cwriter
github.com/vbauerster/mpb/v7/decor
@ -669,7 +670,7 @@ golang.org/x/net/proxy
golang.org/x/net/trace
# golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
golang.org/x/sync/semaphore
# golang.org/x/sys v0.0.0-20210603125802-9665404d3644
# golang.org/x/sys v0.0.0-20210616094352-59db8d763f22
golang.org/x/sys/cpu
golang.org/x/sys/execabs
golang.org/x/sys/internal/unsafeheader