Address CVE-2024-3727

This addrress the CVE-2024-3727 by pulling in the top of main for

c/image, c/common, and c/buildah, all of which have the fix.

Addresses: CVE-2024-3727
No associated Jira cards at the moment.

Signed-off-by: tomsweeneyredhat <tsweeney@redhat.com>
This commit is contained in:
tomsweeneyredhat
2024-05-13 10:55:18 -04:00
parent 0c09421f85
commit 6830d0e3c8
34 changed files with 448 additions and 142 deletions

10
go.mod
View File

@ -13,11 +13,11 @@ require (
github.com/checkpoint-restore/checkpointctl v1.1.0
github.com/checkpoint-restore/go-criu/v7 v7.1.0
github.com/containernetworking/plugins v1.4.1
github.com/containers/buildah v1.35.1-0.20240507152307-453d2fc1099f
github.com/containers/common v0.58.1-0.20240508094622-694258d1f523
github.com/containers/buildah v1.35.1-0.20240510150258-77f239ae12e5
github.com/containers/common v0.58.1-0.20240509172903-2c88a3f280bb
github.com/containers/conmon v2.0.20+incompatible
github.com/containers/gvisor-tap-vsock v0.7.4-0.20240408151405-d744d71db363
github.com/containers/image/v5 v5.30.1-0.20240506211741-f0acff0fe152
github.com/containers/image/v5 v5.30.2-0.20240509191815-9318d0eaaf78
github.com/containers/libhvee v0.7.1
github.com/containers/ocicrypt v1.1.10
github.com/containers/psgo v1.9.0
@ -30,7 +30,7 @@ require (
github.com/cyphar/filepath-securejoin v0.2.5
github.com/digitalocean/go-qemu v0.0.0-20230711162256-2e3d0186973e
github.com/docker/distribution v2.8.3+incompatible
github.com/docker/docker v26.1.1+incompatible
github.com/docker/docker v26.1.2+incompatible
github.com/docker/go-connections v0.5.0
github.com/docker/go-plugins-helpers v0.0.0-20211224144127-6eecb7beb651
github.com/docker/go-units v0.5.0
@ -149,7 +149,7 @@ require (
github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
github.com/hashicorp/go-retryablehttp v0.7.5 // indirect
github.com/hashicorp/go-retryablehttp v0.7.6 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/jinzhu/copier v0.4.0 // indirect
github.com/josharian/intern v1.0.0 // indirect

30
go.sum
View File

@ -77,16 +77,16 @@ github.com/containernetworking/cni v1.1.2 h1:wtRGZVv7olUHMOqouPpn3cXJWpJgM6+EUl3
github.com/containernetworking/cni v1.1.2/go.mod h1:sDpYKmGVENF3s6uvMvGgldDWeG8dMxakj/u+i9ht9vw=
github.com/containernetworking/plugins v1.4.1 h1:+sJRRv8PKhLkXIl6tH1D7RMi+CbbHutDGU+ErLBORWA=
github.com/containernetworking/plugins v1.4.1/go.mod h1:n6FFGKcaY4o2o5msgu/UImtoC+fpQXM3076VHfHbj60=
github.com/containers/buildah v1.35.1-0.20240507152307-453d2fc1099f h1:nkYjiklm9gnopLfp1p+rOucLaLMKs+APUHowVwIk25k=
github.com/containers/buildah v1.35.1-0.20240507152307-453d2fc1099f/go.mod h1:CbpbxJEuDs1ymlLD9057ukMtDOF1HwEuvJaRZAX9Oqs=
github.com/containers/common v0.58.1-0.20240508094622-694258d1f523 h1:cOAggWgjjBmQgkK1kw4La+ov/TooskH8fgI0YUpgksA=
github.com/containers/common v0.58.1-0.20240508094622-694258d1f523/go.mod h1:384OZU759/9Vba1fqnjlngmuBmzCK1UjZY6t1GXsz9Q=
github.com/containers/buildah v1.35.1-0.20240510150258-77f239ae12e5 h1:xtKtw/g2iDkirqSw6Dvvc2ZMPxBYhyN9xPdH81a7hO4=
github.com/containers/buildah v1.35.1-0.20240510150258-77f239ae12e5/go.mod h1:ezOOMchy0Dcu/jKNNsTJbtxvOrhdogVkbG+UxkG77EY=
github.com/containers/common v0.58.1-0.20240509172903-2c88a3f280bb h1:mb5e8J/kErkytiM1J5hqdZENBJfSQyQ37Cgx0hinVYs=
github.com/containers/common v0.58.1-0.20240509172903-2c88a3f280bb/go.mod h1:SCOYkp6ul27v6WoNkbgvhAhhSEM6fYKl2My9/WuESdA=
github.com/containers/conmon v2.0.20+incompatible h1:YbCVSFSCqFjjVwHTPINGdMX1F6JXHGTUje2ZYobNrkg=
github.com/containers/conmon v2.0.20+incompatible/go.mod h1:hgwZ2mtuDrppv78a/cOBNiCm6O0UMWGx1mu7P00nu5I=
github.com/containers/gvisor-tap-vsock v0.7.4-0.20240408151405-d744d71db363 h1:EqWMZeFa08y2c1GniaFkfjlO5AjegoG2foWo6NlDfUY=
github.com/containers/gvisor-tap-vsock v0.7.4-0.20240408151405-d744d71db363/go.mod h1:KN4qqZfwVBzvqlN1Ytbhf84sOzftw+R8YL9bixQlr2Y=
github.com/containers/image/v5 v5.30.1-0.20240506211741-f0acff0fe152 h1:ngQBNZXzTAjGeaba0KhsRjC8uWWWf1pLMhzIAyW2Wts=
github.com/containers/image/v5 v5.30.1-0.20240506211741-f0acff0fe152/go.mod h1:luYQKDGJH9oNAjZj8i6Crnujm24gk2YQbYMAIkAhwNE=
github.com/containers/image/v5 v5.30.2-0.20240509191815-9318d0eaaf78 h1:1fktdUOKdvMbDbAullFBjslw1VewscLwTjsH2S+6ieM=
github.com/containers/image/v5 v5.30.2-0.20240509191815-9318d0eaaf78/go.mod h1:nw5UU0qHFIsg+3cj1u1kP/CmwEioiJrVSDgg1QLhirw=
github.com/containers/libhvee v0.7.1 h1:dWGF5GLq9DZvXo3P8aDp3cNieL5eCaSell4UmeA/jY4=
github.com/containers/libhvee v0.7.1/go.mod h1:fRKB3AyIqHMvq6xaeYhTpckM2cdoq0oecolyoiuLP7M=
github.com/containers/libtrust v0.0.0-20230121012942-c1716e8a8d01 h1:Qzk5C6cYglewc+UyGf6lc8Mj2UaPTHy/iF2De0/77CA=
@ -132,12 +132,12 @@ github.com/disiqueira/gotree/v3 v3.0.2 h1:ik5iuLQQoufZBNPY518dXhiO5056hyNBIK9lWh
github.com/disiqueira/gotree/v3 v3.0.2/go.mod h1:ZuyjE4+mUQZlbpkI24AmruZKhg3VHEgPLDY8Qk+uUu8=
github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
github.com/docker/cli v26.1.1+incompatible h1:bE1/uE2tCa08fMv+7ikLR/RDPoCqytwrLtkIkSzxLvw=
github.com/docker/cli v26.1.1+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
github.com/docker/cli v26.1.2+incompatible h1:/MWZpUMMlr1hCGyquL8QNbL1hbivQ1kLuT3Z9s1Tlpg=
github.com/docker/cli v26.1.2+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk=
github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
github.com/docker/docker v26.1.1+incompatible h1:oI+4kkAgIwwb54b9OC7Xc3hSgu1RlJA/Lln/DF72djQ=
github.com/docker/docker v26.1.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
github.com/docker/docker v26.1.2+incompatible h1:UVX5ZOrrfTGZZYEP+ZDq3Xn9PdHNXaSYMFPDumMqG2k=
github.com/docker/docker v26.1.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
github.com/docker/docker-credential-helpers v0.8.1 h1:j/eKUktUltBtMzKqmfLB0PAgqYyMHOp5vfsD1807oKo=
github.com/docker/docker-credential-helpers v0.8.1/go.mod h1:P3ci7E3lwkZg6XiHdRKft1KckHiO9a2rNtyFbZ/ry9M=
github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c=
@ -286,13 +286,12 @@ github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY
github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ=
github.com/hashicorp/go-hclog v1.5.0 h1:bI2ocEMgcVlz55Oj1xZNBsVi900c7II+fWDyV9o+13c=
github.com/hashicorp/go-hclog v1.5.0/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M=
github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k=
github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M=
github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
github.com/hashicorp/go-retryablehttp v0.7.5 h1:bJj+Pj19UZMIweq/iie+1u5YCdGrnxCT9yvm0e+Nd5M=
github.com/hashicorp/go-retryablehttp v0.7.5/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8=
github.com/hashicorp/go-retryablehttp v0.7.6 h1:TwRYfx2z2C4cLbXmT8I5PgP/xmuqASDyiVuGYfs9GZM=
github.com/hashicorp/go-retryablehttp v0.7.6/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/hugelgupf/p9 v0.3.1-0.20230822151754-54f5c5530921 h1:cfYGdNpXGZobTSSDFB+wx2FRfWptM7sCkScJgVx0Tkk=
github.com/hugelgupf/p9 v0.3.1-0.20230822151754-54f5c5530921/go.mod h1:nMr69J6AmirlSvzeVLK7gj4DUY1oYtSwcSiSJ7BBb0A=
@ -492,7 +491,6 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=

View File

@ -49,10 +49,13 @@ type progressBar struct {
// As a convention, most users of progress bars should call mark100PercentComplete on full success;
// by convention, we don't leave progress bars in partial state when fully done
// (even if we copied much less data than anticipated).
func (c *copier) createProgressBar(pool *mpb.Progress, partial bool, info types.BlobInfo, kind string, onComplete string) *progressBar {
func (c *copier) createProgressBar(pool *mpb.Progress, partial bool, info types.BlobInfo, kind string, onComplete string) (*progressBar, error) {
// shortDigestLen is the length of the digest used for blobs.
const shortDigestLen = 12
if err := info.Digest.Validate(); err != nil { // digest.Digest.Encoded() panics on failure, so validate explicitly.
return nil, err
}
prefix := fmt.Sprintf("Copying %s %s", kind, info.Digest.Encoded())
// Truncate the prefix (chopping of some part of the digest) to make all progress bars aligned in a column.
maxPrefixLen := len("Copying blob ") + shortDigestLen
@ -105,7 +108,7 @@ func (c *copier) createProgressBar(pool *mpb.Progress, partial bool, info types.
return &progressBar{
Bar: bar,
originalSize: info.Size,
}
}, nil
}
// printCopyInfo prints a "Copying ..." message on the copier if the output is

View File

@ -606,7 +606,10 @@ func (ic *imageCopier) copyConfig(ctx context.Context, src types.Image) error {
destInfo, err := func() (types.BlobInfo, error) { // A scope for defer
progressPool := ic.c.newProgressPool()
defer progressPool.Wait()
bar := ic.c.createProgressBar(progressPool, false, srcInfo, "config", "done")
bar, err := ic.c.createProgressBar(progressPool, false, srcInfo, "config", "done")
if err != nil {
return types.BlobInfo{}, err
}
defer bar.Abort(false)
ic.c.printCopyInfo("config", srcInfo)
@ -738,15 +741,21 @@ func (ic *imageCopier) copyLayer(ctx context.Context, srcInfo types.BlobInfo, to
}
if reused {
logrus.Debugf("Skipping blob %s (already present):", srcInfo.Digest)
func() { // A scope for defer
if err := func() error { // A scope for defer
label := "skipped: already exists"
if reusedBlob.MatchedByTOCDigest {
label = "skipped: already exists (found by TOC)"
}
bar := ic.c.createProgressBar(pool, false, types.BlobInfo{Digest: reusedBlob.Digest, Size: 0}, "blob", label)
bar, err := ic.c.createProgressBar(pool, false, types.BlobInfo{Digest: reusedBlob.Digest, Size: 0}, "blob", label)
if err != nil {
return err
}
defer bar.Abort(false)
bar.mark100PercentComplete()
}()
return nil
}(); err != nil {
return types.BlobInfo{}, "", err
}
// Throw an event that the layer has been skipped
if ic.c.options.Progress != nil && ic.c.options.ProgressInterval > 0 {
@ -765,8 +774,11 @@ func (ic *imageCopier) copyLayer(ctx context.Context, srcInfo types.BlobInfo, to
// Attempt a partial only when the source allows to retrieve a blob partially and
// the destination has support for it.
if canAvoidProcessingCompleteLayer && ic.c.rawSource.SupportsGetBlobAt() && ic.c.dest.SupportsPutBlobPartial() {
if reused, blobInfo := func() (bool, types.BlobInfo) { // A scope for defer
bar := ic.c.createProgressBar(pool, true, srcInfo, "blob", "done")
reused, blobInfo, err := func() (bool, types.BlobInfo, error) { // A scope for defer
bar, err := ic.c.createProgressBar(pool, true, srcInfo, "blob", "done")
if err != nil {
return false, types.BlobInfo{}, err
}
hideProgressBar := true
defer func() { // Note that this is not the same as defer bar.Abort(hideProgressBar); we need hideProgressBar to be evaluated lazily.
bar.Abort(hideProgressBar)
@ -789,18 +801,25 @@ func (ic *imageCopier) copyLayer(ctx context.Context, srcInfo types.BlobInfo, to
bar.mark100PercentComplete()
hideProgressBar = false
logrus.Debugf("Retrieved partial blob %v", srcInfo.Digest)
return true, updatedBlobInfoFromUpload(srcInfo, uploadedBlob)
return true, updatedBlobInfoFromUpload(srcInfo, uploadedBlob), nil
}
logrus.Debugf("Failed to retrieve partial blob: %v", err)
return false, types.BlobInfo{}
}(); reused {
return false, types.BlobInfo{}, nil
}()
if err != nil {
return types.BlobInfo{}, "", err
}
if reused {
return blobInfo, cachedDiffID, nil
}
}
// Fallback: copy the layer, computing the diffID if we need to do so
return func() (types.BlobInfo, digest.Digest, error) { // A scope for defer
bar := ic.c.createProgressBar(pool, false, srcInfo, "blob", "done")
bar, err := ic.c.createProgressBar(pool, false, srcInfo, "blob", "done")
if err != nil {
return types.BlobInfo{}, "", err
}
defer bar.Abort(false)
srcStream, srcBlobSize, err := ic.c.rawSource.GetBlob(ctx, srcInfo, ic.c.blobInfoCache)

View File

@ -174,7 +174,10 @@ func (d *dirImageDestination) PutBlobWithOptions(ctx context.Context, stream io.
}
}
blobPath := d.ref.layerPath(blobDigest)
blobPath, err := d.ref.layerPath(blobDigest)
if err != nil {
return private.UploadedBlob{}, err
}
// need to explicitly close the file, since a rename won't otherwise not work on Windows
blobFile.Close()
explicitClosed = true
@ -197,7 +200,10 @@ func (d *dirImageDestination) TryReusingBlobWithOptions(ctx context.Context, inf
if info.Digest == "" {
return false, private.ReusedBlob{}, fmt.Errorf("Can not check for a blob with unknown digest")
}
blobPath := d.ref.layerPath(info.Digest)
blobPath, err := d.ref.layerPath(info.Digest)
if err != nil {
return false, private.ReusedBlob{}, err
}
finfo, err := os.Stat(blobPath)
if err != nil && os.IsNotExist(err) {
return false, private.ReusedBlob{}, nil
@ -217,7 +223,11 @@ func (d *dirImageDestination) TryReusingBlobWithOptions(ctx context.Context, inf
// If the destination is in principle available, refuses this manifest type (e.g. it does not recognize the schema),
// but may accept a different manifest type, the returned error must be an ManifestTypeRejectedError.
func (d *dirImageDestination) PutManifest(ctx context.Context, manifest []byte, instanceDigest *digest.Digest) error {
return os.WriteFile(d.ref.manifestPath(instanceDigest), manifest, 0644)
path, err := d.ref.manifestPath(instanceDigest)
if err != nil {
return err
}
return os.WriteFile(path, manifest, 0644)
}
// PutSignaturesWithFormat writes a set of signatures to the destination.
@ -230,7 +240,11 @@ func (d *dirImageDestination) PutSignaturesWithFormat(ctx context.Context, signa
if err != nil {
return err
}
if err := os.WriteFile(d.ref.signaturePath(i, instanceDigest), blob, 0644); err != nil {
path, err := d.ref.signaturePath(i, instanceDigest)
if err != nil {
return err
}
if err := os.WriteFile(path, blob, 0644); err != nil {
return err
}
}

View File

@ -55,7 +55,11 @@ func (s *dirImageSource) Close() error {
// If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve (when the primary manifest is a manifest list);
// this never happens if the primary manifest is not a manifest list (e.g. if the source never returns manifest lists).
func (s *dirImageSource) GetManifest(ctx context.Context, instanceDigest *digest.Digest) ([]byte, string, error) {
m, err := os.ReadFile(s.ref.manifestPath(instanceDigest))
path, err := s.ref.manifestPath(instanceDigest)
if err != nil {
return nil, "", err
}
m, err := os.ReadFile(path)
if err != nil {
return nil, "", err
}
@ -66,7 +70,11 @@ func (s *dirImageSource) GetManifest(ctx context.Context, instanceDigest *digest
// The Digest field in BlobInfo is guaranteed to be provided, Size may be -1 and MediaType may be optionally provided.
// May update BlobInfoCache, preferably after it knows for certain that a blob truly exists at a specific location.
func (s *dirImageSource) GetBlob(ctx context.Context, info types.BlobInfo, cache types.BlobInfoCache) (io.ReadCloser, int64, error) {
r, err := os.Open(s.ref.layerPath(info.Digest))
path, err := s.ref.layerPath(info.Digest)
if err != nil {
return nil, -1, err
}
r, err := os.Open(path)
if err != nil {
return nil, -1, err
}
@ -84,7 +92,10 @@ func (s *dirImageSource) GetBlob(ctx context.Context, info types.BlobInfo, cache
func (s *dirImageSource) GetSignaturesWithFormat(ctx context.Context, instanceDigest *digest.Digest) ([]signature.Signature, error) {
signatures := []signature.Signature{}
for i := 0; ; i++ {
path := s.ref.signaturePath(i, instanceDigest)
path, err := s.ref.signaturePath(i, instanceDigest)
if err != nil {
return nil, err
}
sigBlob, err := os.ReadFile(path)
if err != nil {
if os.IsNotExist(err) {

View File

@ -161,25 +161,34 @@ func (ref dirReference) DeleteImage(ctx context.Context, sys *types.SystemContex
}
// manifestPath returns a path for the manifest within a directory using our conventions.
func (ref dirReference) manifestPath(instanceDigest *digest.Digest) string {
func (ref dirReference) manifestPath(instanceDigest *digest.Digest) (string, error) {
if instanceDigest != nil {
return filepath.Join(ref.path, instanceDigest.Encoded()+".manifest.json")
if err := instanceDigest.Validate(); err != nil { // digest.Digest.Encoded() panics on failure, and could possibly result in a path with ../, so validate explicitly.
return "", err
}
return filepath.Join(ref.path, instanceDigest.Encoded()+".manifest.json"), nil
}
return filepath.Join(ref.path, "manifest.json")
return filepath.Join(ref.path, "manifest.json"), nil
}
// layerPath returns a path for a layer tarball within a directory using our conventions.
func (ref dirReference) layerPath(digest digest.Digest) string {
func (ref dirReference) layerPath(digest digest.Digest) (string, error) {
if err := digest.Validate(); err != nil { // digest.Digest.Encoded() panics on failure, and could possibly result in a path with ../, so validate explicitly.
return "", err
}
// FIXME: Should we keep the digest identification?
return filepath.Join(ref.path, digest.Encoded())
return filepath.Join(ref.path, digest.Encoded()), nil
}
// signaturePath returns a path for a signature within a directory using our conventions.
func (ref dirReference) signaturePath(index int, instanceDigest *digest.Digest) string {
func (ref dirReference) signaturePath(index int, instanceDigest *digest.Digest) (string, error) {
if instanceDigest != nil {
return filepath.Join(ref.path, fmt.Sprintf(instanceDigest.Encoded()+".signature-%d", index+1))
if err := instanceDigest.Validate(); err != nil { // digest.Digest.Encoded() panics on failure, and could possibly result in a path with ../, so validate explicitly.
return "", err
}
return filepath.Join(ref.path, fmt.Sprintf(instanceDigest.Encoded()+".signature-%d", index+1)), nil
}
return filepath.Join(ref.path, fmt.Sprintf("signature-%d", index+1))
return filepath.Join(ref.path, fmt.Sprintf("signature-%d", index+1)), nil
}
// versionPath returns a path for the version file within a directory using our conventions.

View File

@ -963,6 +963,8 @@ func (c *dockerClient) detectProperties(ctx context.Context) error {
return c.detectPropertiesError
}
// fetchManifest fetches a manifest for (the repo of ref) + tagOrDigest.
// The caller is responsible for ensuring tagOrDigest uses the expected format.
func (c *dockerClient) fetchManifest(ctx context.Context, ref dockerReference, tagOrDigest string) ([]byte, string, error) {
path := fmt.Sprintf(manifestPath, reference.Path(ref.ref), tagOrDigest)
headers := map[string][]string{
@ -1042,6 +1044,9 @@ func (c *dockerClient) getBlob(ctx context.Context, ref dockerReference, info ty
}
}
if err := info.Digest.Validate(); err != nil { // Make sure info.Digest.String() does not contain any unexpected characters
return nil, 0, err
}
path := fmt.Sprintf(blobsPath, reference.Path(ref.ref), info.Digest.String())
logrus.Debugf("Downloading %s", path)
res, err := c.makeRequest(ctx, http.MethodGet, path, nil, nil, v2Auth, nil)
@ -1105,7 +1110,10 @@ func isManifestUnknownError(err error) bool {
// digest in ref.
// It returns (nil, nil) if the manifest does not exist.
func (c *dockerClient) getSigstoreAttachmentManifest(ctx context.Context, ref dockerReference, digest digest.Digest) (*manifest.OCI1, error) {
tag := sigstoreAttachmentTag(digest)
tag, err := sigstoreAttachmentTag(digest)
if err != nil {
return nil, err
}
sigstoreRef, err := reference.WithTag(reference.TrimNamed(ref.ref), tag)
if err != nil {
return nil, err
@ -1138,6 +1146,9 @@ func (c *dockerClient) getSigstoreAttachmentManifest(ctx context.Context, ref do
// getExtensionsSignatures returns signatures from the X-Registry-Supports-Signatures API extension,
// using the original data structures.
func (c *dockerClient) getExtensionsSignatures(ctx context.Context, ref dockerReference, manifestDigest digest.Digest) (*extensionSignatureList, error) {
if err := manifestDigest.Validate(); err != nil { // Make sure manifestDigest.String() does not contain any unexpected characters
return nil, err
}
path := fmt.Sprintf(extensionsSignaturePath, reference.Path(ref.ref), manifestDigest)
res, err := c.makeRequest(ctx, http.MethodGet, path, nil, nil, v2Auth, nil)
if err != nil {
@ -1161,8 +1172,11 @@ func (c *dockerClient) getExtensionsSignatures(ctx context.Context, ref dockerRe
}
// sigstoreAttachmentTag returns a sigstore attachment tag for the specified digest.
func sigstoreAttachmentTag(d digest.Digest) string {
return strings.Replace(d.String(), ":", "-", 1) + ".sig"
func sigstoreAttachmentTag(d digest.Digest) (string, error) {
if err := d.Validate(); err != nil { // Make sure d.String() doesnt contain any unexpected characters
return "", err
}
return strings.Replace(d.String(), ":", "-", 1) + ".sig", nil
}
// Close removes resources associated with an initialized dockerClient, if any.

View File

@ -88,7 +88,12 @@ func GetRepositoryTags(ctx context.Context, sys *types.SystemContext, ref types.
if err = json.NewDecoder(res.Body).Decode(&tagsHolder); err != nil {
return nil, err
}
tags = append(tags, tagsHolder.Tags...)
for _, tag := range tagsHolder.Tags {
if _, err := reference.WithTag(dr.ref, tag); err != nil { // Ensure the tag does not contain unexpected values
return nil, fmt.Errorf("registry returned invalid tag %q: %w", tag, err)
}
tags = append(tags, tag)
}
link := res.Header.Get("Link")
if link == "" {

View File

@ -230,6 +230,9 @@ func (d *dockerImageDestination) PutBlobWithOptions(ctx context.Context, stream
// If the destination does not contain the blob, or it is unknown, blobExists ordinarily returns (false, -1, nil);
// it returns a non-nil error only on an unexpected failure.
func (d *dockerImageDestination) blobExists(ctx context.Context, repo reference.Named, digest digest.Digest, extraScope *authScope) (bool, int64, error) {
if err := digest.Validate(); err != nil { // Make sure digest.String() does not contain any unexpected characters
return false, -1, err
}
checkPath := fmt.Sprintf(blobsPath, reference.Path(repo), digest.String())
logrus.Debugf("Checking %s", checkPath)
res, err := d.c.makeRequest(ctx, http.MethodHead, checkPath, nil, nil, v2Auth, extraScope)
@ -458,6 +461,7 @@ func (d *dockerImageDestination) PutManifest(ctx context.Context, m []byte, inst
// particular instance.
refTail = instanceDigest.String()
// Double-check that the manifest we've been given matches the digest we've been given.
// This also validates the format of instanceDigest.
matches, err := manifest.MatchesDigest(m, *instanceDigest)
if err != nil {
return fmt.Errorf("digesting manifest in PutManifest: %w", err)
@ -624,11 +628,13 @@ func (d *dockerImageDestination) putSignaturesToLookaside(signatures []signature
// NOTE: Keep this in sync with docs/signature-protocols.md!
for i, signature := range signatures {
sigURL := lookasideStorageURL(d.c.signatureBase, manifestDigest, i)
err := d.putOneSignature(sigURL, signature)
sigURL, err := lookasideStorageURL(d.c.signatureBase, manifestDigest, i)
if err != nil {
return err
}
if err := d.putOneSignature(sigURL, signature); err != nil {
return err
}
}
// Remove any other signatures, if present.
// We stop at the first missing signature; if a previous deleting loop aborted
@ -636,7 +642,10 @@ func (d *dockerImageDestination) putSignaturesToLookaside(signatures []signature
// is enough for dockerImageSource to stop looking for other signatures, so that
// is sufficient.
for i := len(signatures); ; i++ {
sigURL := lookasideStorageURL(d.c.signatureBase, manifestDigest, i)
sigURL, err := lookasideStorageURL(d.c.signatureBase, manifestDigest, i)
if err != nil {
return err
}
missing, err := d.c.deleteOneSignature(sigURL)
if err != nil {
return err
@ -767,8 +776,12 @@ func (d *dockerImageDestination) putSignaturesToSigstoreAttachments(ctx context.
if err != nil {
return err
}
attachmentTag, err := sigstoreAttachmentTag(manifestDigest)
if err != nil {
return err
}
logrus.Debugf("Uploading sigstore attachment manifest")
return d.uploadManifest(ctx, manifestBlob, sigstoreAttachmentTag(manifestDigest))
return d.uploadManifest(ctx, manifestBlob, attachmentTag)
}
func layerMatchesSigstoreSignature(layer imgspecv1.Descriptor, mimeType string,
@ -884,6 +897,7 @@ func (d *dockerImageDestination) putSignaturesToAPIExtension(ctx context.Context
return err
}
// manifestDigest is known to be valid because it was not rejected by getExtensionsSignatures above.
path := fmt.Sprintf(extensionsSignaturePath, reference.Path(d.ref.ref), manifestDigest.String())
res, err := d.c.makeRequest(ctx, http.MethodPut, path, nil, bytes.NewReader(body), v2Auth, nil)
if err != nil {

View File

@ -194,6 +194,9 @@ func simplifyContentType(contentType string) string {
// this never happens if the primary manifest is not a manifest list (e.g. if the source never returns manifest lists).
func (s *dockerImageSource) GetManifest(ctx context.Context, instanceDigest *digest.Digest) ([]byte, string, error) {
if instanceDigest != nil {
if err := instanceDigest.Validate(); err != nil { // Make sure instanceDigest.String() does not contain any unexpected characters
return nil, "", err
}
return s.fetchManifest(ctx, instanceDigest.String())
}
err := s.ensureManifestIsLoaded(ctx)
@ -203,6 +206,8 @@ func (s *dockerImageSource) GetManifest(ctx context.Context, instanceDigest *dig
return s.cachedManifest, s.cachedManifestMIMEType, nil
}
// fetchManifest fetches a manifest for tagOrDigest.
// The caller is responsible for ensuring tagOrDigest uses the expected format.
func (s *dockerImageSource) fetchManifest(ctx context.Context, tagOrDigest string) ([]byte, string, error) {
return s.c.fetchManifest(ctx, s.physicalRef, tagOrDigest)
}
@ -352,6 +357,9 @@ func (s *dockerImageSource) GetBlobAt(ctx context.Context, info types.BlobInfo,
return nil, nil, fmt.Errorf("external URLs not supported with GetBlobAt")
}
if err := info.Digest.Validate(); err != nil { // Make sure info.Digest.String() does not contain any unexpected characters
return nil, nil, err
}
path := fmt.Sprintf(blobsPath, reference.Path(s.physicalRef.ref), info.Digest.String())
logrus.Debugf("Downloading %s", path)
res, err := s.c.makeRequest(ctx, http.MethodGet, path, headers, nil, v2Auth, nil)
@ -462,7 +470,10 @@ func (s *dockerImageSource) getSignaturesFromLookaside(ctx context.Context, inst
return nil, fmt.Errorf("server provided %d signatures, assuming that's unreasonable and a server error", maxLookasideSignatures)
}
sigURL := lookasideStorageURL(s.c.signatureBase, manifestDigest, i)
sigURL, err := lookasideStorageURL(s.c.signatureBase, manifestDigest, i)
if err != nil {
return nil, err
}
signature, missing, err := s.getOneSignature(ctx, sigURL)
if err != nil {
return nil, err
@ -660,7 +671,10 @@ func deleteImage(ctx context.Context, sys *types.SystemContext, ref dockerRefere
}
for i := 0; ; i++ {
sigURL := lookasideStorageURL(c.signatureBase, manifestDigest, i)
sigURL, err := lookasideStorageURL(c.signatureBase, manifestDigest, i)
if err != nil {
return err
}
missing, err := c.deleteOneSignature(sigURL)
if err != nil {
return err

View File

@ -111,11 +111,19 @@ func (d *Destination) PutBlobWithOptions(ctx context.Context, stream io.Reader,
return private.UploadedBlob{}, fmt.Errorf("reading Config file stream: %w", err)
}
d.config = buf
if err := d.archive.sendFileLocked(d.archive.configPath(inputInfo.Digest), inputInfo.Size, bytes.NewReader(buf)); err != nil {
configPath, err := d.archive.configPath(inputInfo.Digest)
if err != nil {
return private.UploadedBlob{}, err
}
if err := d.archive.sendFileLocked(configPath, inputInfo.Size, bytes.NewReader(buf)); err != nil {
return private.UploadedBlob{}, fmt.Errorf("writing Config file: %w", err)
}
} else {
if err := d.archive.sendFileLocked(d.archive.physicalLayerPath(inputInfo.Digest), inputInfo.Size, stream); err != nil {
layerPath, err := d.archive.physicalLayerPath(inputInfo.Digest)
if err != nil {
return private.UploadedBlob{}, err
}
if err := d.archive.sendFileLocked(layerPath, inputInfo.Size, stream); err != nil {
return private.UploadedBlob{}, err
}
}

View File

@ -95,7 +95,10 @@ func (w *Writer) ensureSingleLegacyLayerLocked(layerID string, layerDigest diges
if !w.legacyLayers.Contains(layerID) {
// Create a symlink for the legacy format, where there is one subdirectory per layer ("image").
// See also the comment in physicalLayerPath.
physicalLayerPath := w.physicalLayerPath(layerDigest)
physicalLayerPath, err := w.physicalLayerPath(layerDigest)
if err != nil {
return err
}
if err := w.sendSymlinkLocked(filepath.Join(layerID, legacyLayerFileName), filepath.Join("..", physicalLayerPath)); err != nil {
return fmt.Errorf("creating layer symbolic link: %w", err)
}
@ -139,6 +142,9 @@ func (w *Writer) writeLegacyMetadataLocked(layerDescriptors []manifest.Schema2De
}
// This chainID value matches the computation in docker/docker/layer.CreateChainID …
if err := l.Digest.Validate(); err != nil { // This should never fail on this code path, still: make sure the chainID computation is unambiguous.
return err
}
if chainID == "" {
chainID = l.Digest
} else {
@ -204,12 +210,20 @@ func checkManifestItemsMatch(a, b *ManifestItem) error {
func (w *Writer) ensureManifestItemLocked(layerDescriptors []manifest.Schema2Descriptor, configDigest digest.Digest, repoTags []reference.NamedTagged) error {
layerPaths := []string{}
for _, l := range layerDescriptors {
layerPaths = append(layerPaths, w.physicalLayerPath(l.Digest))
p, err := w.physicalLayerPath(l.Digest)
if err != nil {
return err
}
layerPaths = append(layerPaths, p)
}
var item *ManifestItem
configPath, err := w.configPath(configDigest)
if err != nil {
return err
}
newItem := ManifestItem{
Config: w.configPath(configDigest),
Config: configPath,
RepoTags: []string{},
Layers: layerPaths,
Parent: "", // We dont have this information
@ -294,21 +308,27 @@ func (w *Writer) Close() error {
// configPath returns a path we choose for storing a config with the specified digest.
// NOTE: This is an internal implementation detail, not a format property, and can change
// any time.
func (w *Writer) configPath(configDigest digest.Digest) string {
return configDigest.Hex() + ".json"
func (w *Writer) configPath(configDigest digest.Digest) (string, error) {
if err := configDigest.Validate(); err != nil { // digest.Digest.Hex() panics on failure, and could possibly result in unexpected paths, so validate explicitly.
return "", err
}
return configDigest.Hex() + ".json", nil
}
// physicalLayerPath returns a path we choose for storing a layer with the specified digest
// (the actual path, i.e. a regular file, not a symlink that may be used in the legacy format).
// NOTE: This is an internal implementation detail, not a format property, and can change
// any time.
func (w *Writer) physicalLayerPath(layerDigest digest.Digest) string {
func (w *Writer) physicalLayerPath(layerDigest digest.Digest) (string, error) {
if err := layerDigest.Validate(); err != nil { // digest.Digest.Hex() panics on failure, and could possibly result in unexpected paths, so validate explicitly.
return "", err
}
// Note that this can't be e.g. filepath.Join(l.Digest.Hex(), legacyLayerFileName); due to the way
// writeLegacyMetadata constructs layer IDs differently from inputinfo.Digest values (as described
// inside it), most of the layers would end up in subdirectories alone without any metadata; (docker load)
// tries to load every subdirectory as an image and fails if the config is missing. So, keep the layers
// in the root of the tarball.
return layerDigest.Hex() + ".tar"
return layerDigest.Hex() + ".tar", nil
}
type tarFI struct {

View File

@ -287,8 +287,11 @@ func (ns registryNamespace) signatureTopLevel(write bool) string {
// lookasideStorageURL returns an URL usable for accessing signature index in base with known manifestDigest.
// base is not nil from the caller
// NOTE: Keep this in sync with docs/signature-protocols.md!
func lookasideStorageURL(base lookasideStorageBase, manifestDigest digest.Digest, index int) *url.URL {
func lookasideStorageURL(base lookasideStorageBase, manifestDigest digest.Digest, index int) (*url.URL, error) {
if err := manifestDigest.Validate(); err != nil { // digest.Digest.Hex() panics on failure, and could possibly result in a path with ../, so validate explicitly.
return nil, err
}
sigURL := *base
sigURL.Path = fmt.Sprintf("%s@%s=%s/signature-%d", sigURL.Path, manifestDigest.Algorithm(), manifestDigest.Hex(), index+1)
return &sigURL
return &sigURL, nil
}

View File

@ -109,6 +109,9 @@ func (s *openshiftImageSource) GetSignaturesWithFormat(ctx context.Context, inst
}
imageStreamImageName = s.imageStreamImageName
} else {
if err := instanceDigest.Validate(); err != nil { // Make sure instanceDigest.String() does not contain any unexpected characters
return nil, err
}
imageStreamImageName = instanceDigest.String()
}
image, err := s.client.getImage(ctx, imageStreamImageName)

View File

@ -347,6 +347,10 @@ func (d *ostreeImageDestination) TryReusingBlobWithOptions(ctx context.Context,
}
d.repo = repo
}
if err := info.Digest.Validate(); err != nil { // digest.Digest.Hex() panics on failure, so validate explicitly.
return false, private.ReusedBlob{}, err
}
branch := fmt.Sprintf("ociimage/%s", info.Digest.Hex())
found, data, err := readMetadata(d.repo, branch, "docker.uncompressed_digest")
@ -472,12 +476,18 @@ func (d *ostreeImageDestination) Commit(context.Context, types.UnparsedImage) er
return nil
}
for _, layer := range d.schema.LayersDescriptors {
if err := layer.Digest.Validate(); err != nil { // digest.Digest.Encoded() panics on failure, so validate explicitly.
return err
}
hash := layer.Digest.Hex()
if err = checkLayer(hash); err != nil {
return err
}
}
for _, layer := range d.schema.FSLayers {
if err := layer.BlobSum.Validate(); err != nil { // digest.Digest.Encoded() panics on failure, so validate explicitly.
return err
}
hash := layer.BlobSum.Hex()
if err = checkLayer(hash); err != nil {
return err

View File

@ -286,7 +286,9 @@ func (s *ostreeImageSource) readSingleFile(commit, path string) (io.ReadCloser,
// The Digest field in BlobInfo is guaranteed to be provided, Size may be -1 and MediaType may be optionally provided.
// May update BlobInfoCache, preferably after it knows for certain that a blob truly exists at a specific location.
func (s *ostreeImageSource) GetBlob(ctx context.Context, info types.BlobInfo, cache types.BlobInfoCache) (io.ReadCloser, int64, error) {
if err := info.Digest.Validate(); err != nil { // digest.Digest.Encoded() panics on failure, so validate explicitly.
return nil, -1, err
}
blob := info.Digest.Hex()
// Ensure s.compressed is initialized. It is build by LayerInfosForCopy.

View File

@ -78,12 +78,15 @@ func (b *BlobCache) DeleteImage(ctx context.Context, sys *types.SystemContext) e
}
// blobPath returns the path appropriate for storing a blob with digest.
func (b *BlobCache) blobPath(digest digest.Digest, isConfig bool) string {
func (b *BlobCache) blobPath(digest digest.Digest, isConfig bool) (string, error) {
if err := digest.Validate(); err != nil { // Make sure digest.String() does not contain any unexpected characters
return "", err
}
baseName := digest.String()
if isConfig {
baseName += ".config"
}
return filepath.Join(b.directory, baseName)
return filepath.Join(b.directory, baseName), nil
}
// findBlob checks if we have a blob for info in cache (whether a config or not)
@ -95,7 +98,10 @@ func (b *BlobCache) findBlob(info types.BlobInfo) (string, int64, bool, error) {
}
for _, isConfig := range []bool{false, true} {
path := b.blobPath(info.Digest, isConfig)
path, err := b.blobPath(info.Digest, isConfig)
if err != nil {
return "", -1, false, err
}
fileInfo, err := os.Stat(path)
if err == nil && (info.Size == -1 || info.Size == fileInfo.Size()) {
return path, fileInfo.Size(), isConfig, nil

View File

@ -114,7 +114,10 @@ func (d *blobCacheDestination) saveStream(wg *sync.WaitGroup, decompressReader i
}
// Determine the name that we should give to the uncompressed copy of the blob.
decompressedFilename := d.reference.blobPath(digester.Digest(), isConfig)
decompressedFilename, err := d.reference.blobPath(digester.Digest(), isConfig)
if err != nil {
return
}
// Rename the temporary file.
if err := os.Rename(tempFile.Name(), decompressedFilename); err != nil {
logrus.Debugf("error renaming new decompressed copy of blob %q into place at %q: %v", digester.Digest().String(), decompressedFilename, err)
@ -152,7 +155,10 @@ func (d *blobCacheDestination) PutBlobWithOptions(ctx context.Context, stream io
needToWait := false
compression := archive.Uncompressed
if inputInfo.Digest != "" {
filename := d.reference.blobPath(inputInfo.Digest, options.IsConfig)
filename, err2 := d.reference.blobPath(inputInfo.Digest, options.IsConfig)
if err2 != nil {
return private.UploadedBlob{}, err2
}
tempfile, err = os.CreateTemp(filepath.Dir(filename), filepath.Base(filename))
if err == nil {
stream = io.TeeReader(stream, tempfile)
@ -281,7 +287,10 @@ func (d *blobCacheDestination) PutManifest(ctx context.Context, manifestBytes []
if err != nil {
logrus.Warnf("error digesting manifest %q: %v", string(manifestBytes), err)
} else {
filename := d.reference.blobPath(manifestDigest, false)
filename, err := d.reference.blobPath(manifestDigest, false)
if err != nil {
return err
}
if err = ioutils.AtomicWriteFile(filename, manifestBytes, 0600); err != nil {
logrus.Warnf("error saving manifest as %q: %v", filename, err)
}

View File

@ -56,7 +56,10 @@ func (s *blobCacheSource) Close() error {
func (s *blobCacheSource) GetManifest(ctx context.Context, instanceDigest *digest.Digest) ([]byte, string, error) {
if instanceDigest != nil {
filename := s.reference.blobPath(*instanceDigest, false)
filename, err := s.reference.blobPath(*instanceDigest, false)
if err != nil {
return nil, "", err
}
manifestBytes, err := os.ReadFile(filename)
if err == nil {
s.cacheHits++
@ -136,8 +139,10 @@ func (s *blobCacheSource) LayerInfosForCopy(ctx context.Context, instanceDigest
replacedInfos := make([]types.BlobInfo, 0, len(infos))
for _, info := range infos {
var replaceDigest []byte
var err error
blobFile := s.reference.blobPath(info.Digest, false)
blobFile, err := s.reference.blobPath(info.Digest, false)
if err != nil {
return nil, err
}
var alternate string
switch s.reference.compress {
case types.Compress:
@ -148,7 +153,10 @@ func (s *blobCacheSource) LayerInfosForCopy(ctx context.Context, instanceDigest
replaceDigest, err = os.ReadFile(alternate)
}
if err == nil && digest.Digest(replaceDigest).Validate() == nil {
alternate = s.reference.blobPath(digest.Digest(replaceDigest), false)
alternate, err = s.reference.blobPath(digest.Digest(replaceDigest), false)
if err != nil {
return nil, err
}
fileInfo, err := os.Stat(alternate)
if err == nil {
switch info.MediaType {

View File

@ -361,6 +361,18 @@ func (s *storageImageDestination) TryReusingBlobWithOptions(ctx context.Context,
// tryReusingBlobAsPending implements TryReusingBlobWithOptions for (blobDigest, size or -1), filling s.blobDiffIDs and other metadata.
// The caller must arrange the blob to be eventually committed using s.commitLayer().
func (s *storageImageDestination) tryReusingBlobAsPending(blobDigest digest.Digest, size int64, options *private.TryReusingBlobOptions) (bool, private.ReusedBlob, error) {
if blobDigest == "" {
return false, private.ReusedBlob{}, errors.New(`Can not check for a blob with unknown digest`)
}
if err := blobDigest.Validate(); err != nil {
return false, private.ReusedBlob{}, fmt.Errorf("Can not check for a blob with invalid digest: %w", err)
}
if options.TOCDigest != "" {
if err := options.TOCDigest.Validate(); err != nil {
return false, private.ReusedBlob{}, fmt.Errorf("Can not check for a blob with invalid digest: %w", err)
}
}
// lock the entire method as it executes fairly quickly
s.lock.Lock()
defer s.lock.Unlock()
@ -380,18 +392,6 @@ func (s *storageImageDestination) tryReusingBlobAsPending(blobDigest digest.Dige
}
}
if blobDigest == "" {
return false, private.ReusedBlob{}, errors.New(`Can not check for a blob with unknown digest`)
}
if err := blobDigest.Validate(); err != nil {
return false, private.ReusedBlob{}, fmt.Errorf("Can not check for a blob with invalid digest: %w", err)
}
if options.TOCDigest != "" {
if err := options.TOCDigest.Validate(); err != nil {
return false, private.ReusedBlob{}, fmt.Errorf("Can not check for a blob with invalid digest: %w", err)
}
}
// Check if we have a wasn't-compressed layer in storage that's based on that blob.
// Check if we've already cached it in a file.
@ -1069,8 +1069,12 @@ func (s *storageImageDestination) Commit(ctx context.Context, unparsedToplevel t
if err != nil {
return fmt.Errorf("digesting top-level manifest: %w", err)
}
key, err := manifestBigDataKey(manifestDigest)
if err != nil {
return err
}
options.BigData = append(options.BigData, storage.ImageBigDataOption{
Key: manifestBigDataKey(manifestDigest),
Key: key,
Data: toplevelManifest,
Digest: manifestDigest,
})
@ -1078,8 +1082,12 @@ func (s *storageImageDestination) Commit(ctx context.Context, unparsedToplevel t
// Set up to save the image's manifest. Allow looking it up by digest by using the key convention defined by the Store.
// Record the manifest twice: using a digest-specific key to allow references to that specific digest instance,
// and using storage.ImageDigestBigDataKey for future users that dont specify any digest and for compatibility with older readers.
key, err := manifestBigDataKey(s.manifestDigest)
if err != nil {
return err
}
options.BigData = append(options.BigData, storage.ImageBigDataOption{
Key: manifestBigDataKey(s.manifestDigest),
Key: key,
Data: s.manifest,
Digest: s.manifestDigest,
})
@ -1097,8 +1105,12 @@ func (s *storageImageDestination) Commit(ctx context.Context, unparsedToplevel t
})
}
for instanceDigest, signatures := range s.signatureses {
key, err := signatureBigDataKey(instanceDigest)
if err != nil {
return err
}
options.BigData = append(options.BigData, storage.ImageBigDataOption{
Key: signatureBigDataKey(instanceDigest),
Key: key,
Data: signatures,
Digest: digest.Canonical.FromBytes(signatures),
})

View File

@ -21,14 +21,20 @@ var (
// manifestBigDataKey returns a key suitable for recording a manifest with the specified digest using storage.Store.ImageBigData and related functions.
// If a specific manifest digest is explicitly requested by the user, the key returned by this function should be used preferably;
// for compatibility, if a manifest is not available under this key, check also storage.ImageDigestBigDataKey
func manifestBigDataKey(digest digest.Digest) string {
return storage.ImageDigestManifestBigDataNamePrefix + "-" + digest.String()
func manifestBigDataKey(digest digest.Digest) (string, error) {
if err := digest.Validate(); err != nil { // Make sure info.Digest.String() uses the expected format and does not collide with other BigData keys.
return "", err
}
return storage.ImageDigestManifestBigDataNamePrefix + "-" + digest.String(), nil
}
// signatureBigDataKey returns a key suitable for recording the signatures associated with the manifest with the specified digest using storage.Store.ImageBigData and related functions.
// If a specific manifest digest is explicitly requested by the user, the key returned by this function should be used preferably;
func signatureBigDataKey(digest digest.Digest) string {
return "signature-" + digest.Encoded()
func signatureBigDataKey(digest digest.Digest) (string, error) {
if err := digest.Validate(); err != nil { // digest.Digest.Encoded() panics on failure, so validate explicitly.
return "", err
}
return "signature-" + digest.Encoded(), nil
}
// storageImageMetadata is stored, as JSON, in storage.Image.Metadata

View File

@ -73,7 +73,10 @@ func multiArchImageMatchesSystemContext(store storage.Store, img *storage.Image,
// We don't need to care about storage.ImageDigestBigDataKey because
// manifests lists are only stored into storage by c/image versions
// that know about manifestBigDataKey, and only using that key.
key := manifestBigDataKey(manifestDigest)
key, err := manifestBigDataKey(manifestDigest)
if err != nil {
return false // This should never happen, manifestDigest comes from a reference.Digested, and that validates the format.
}
manifestBytes, err := store.ImageBigData(img.ID, key)
if err != nil {
return false
@ -95,7 +98,10 @@ func multiArchImageMatchesSystemContext(store storage.Store, img *storage.Image,
if err != nil {
return false
}
key = manifestBigDataKey(chosenInstance)
key, err = manifestBigDataKey(chosenInstance)
if err != nil {
return false
}
_, err = store.ImageBigData(img.ID, key)
return err == nil // true if img.ID is based on chosenInstance.
}

View File

@ -237,7 +237,10 @@ func (s *storageImageSource) getBlobAndLayerID(digest digest.Digest, layers []st
// GetManifest() reads the image's manifest.
func (s *storageImageSource) GetManifest(ctx context.Context, instanceDigest *digest.Digest) (manifestBlob []byte, mimeType string, err error) {
if instanceDigest != nil {
key := manifestBigDataKey(*instanceDigest)
key, err := manifestBigDataKey(*instanceDigest)
if err != nil {
return nil, "", err
}
blob, err := s.imageRef.transport.store.ImageBigData(s.image.ID, key)
if err != nil {
return nil, "", fmt.Errorf("reading manifest for image instance %q: %w", *instanceDigest, err)
@ -249,7 +252,10 @@ func (s *storageImageSource) GetManifest(ctx context.Context, instanceDigest *di
// Prefer the manifest corresponding to the user-specified digest, if available.
if s.imageRef.named != nil {
if digested, ok := s.imageRef.named.(reference.Digested); ok {
key := manifestBigDataKey(digested.Digest())
key, err := manifestBigDataKey(digested.Digest())
if err != nil {
return nil, "", err
}
blob, err := s.imageRef.transport.store.ImageBigData(s.image.ID, key)
if err != nil && !os.IsNotExist(err) { // os.IsNotExist is true if the image exists but there is no data corresponding to key
return nil, "", err
@ -385,7 +391,14 @@ func (s *storageImageSource) GetSignaturesWithFormat(ctx context.Context, instan
instance := "default instance"
if instanceDigest != nil {
signatureSizes = s.metadata.SignaturesSizes[*instanceDigest]
key = signatureBigDataKey(*instanceDigest)
k, err := signatureBigDataKey(*instanceDigest)
if err != nil {
return nil, err
}
key = k
if err := instanceDigest.Validate(); err != nil { // digest.Digest.Encoded() panics on failure, so validate explicitly.
return nil, err
}
instance = instanceDigest.Encoded()
}
if len(signatureSizes) > 0 {

View File

@ -8,7 +8,7 @@ const (
// VersionMinor is for functionality in a backwards-compatible manner
VersionMinor = 30
// VersionPatch is for backwards-compatible bug fixes
VersionPatch = 1
VersionPatch = 2
// VersionDev indicates development branch. Releases will be empty string.
VersionDev = "-dev"

View File

@ -0,0 +1 @@
1.22.2

View File

@ -1,14 +1,26 @@
## 0.7.6 (May 9, 2024)
ENHANCEMENTS:
- client: support a `RetryPrepare` function for modifying the request before retrying (#216)
- client: support HTTP-date values for `Retry-After` header value (#138)
- client: avoid reading entire body when the body is a `*bytes.Reader` (#197)
BUG FIXES:
- client: fix a broken check for invalid server certificate in go 1.20+ (#210)
## 0.7.5 (Nov 8, 2023)
BUG FIXES
BUG FIXES:
- client: fixes an issue where the request body is not preserved on temporary redirects or re-established HTTP/2 connections [GH-207]
- client: fixes an issue where the request body is not preserved on temporary redirects or re-established HTTP/2 connections (#207)
## 0.7.4 (Jun 6, 2023)
BUG FIXES
BUG FIXES:
- client: fixing an issue where the Content-Type header wouldn't be sent with an empty payload when using HTTP/2 [GH-194]
- client: fixing an issue where the Content-Type header wouldn't be sent with an empty payload when using HTTP/2 (#194)
## 0.7.3 (May 15, 2023)

View File

@ -1 +1 @@
* @hashicorp/release-engineering
* @hashicorp/go-retryablehttp-maintainers

View File

@ -2,7 +2,7 @@ default: test
test:
go vet ./...
go test -race ./...
go test -v -race ./...
updatedeps:
go get -f -t -u ./...

View File

@ -59,4 +59,4 @@ standardClient := retryClient.StandardClient() // *http.Client
```
For more usage and examples see the
[godoc](http://godoc.org/github.com/hashicorp/go-retryablehttp).
[pkg.go.dev](https://pkg.go.dev/github.com/hashicorp/go-retryablehttp).

View File

@ -0,0 +1,14 @@
// Copyright (c) HashiCorp, Inc.
// SPDX-License-Identifier: MPL-2.0
//go:build !go1.20
// +build !go1.20
package retryablehttp
import "crypto/x509"
func isCertError(err error) bool {
_, ok := err.(x509.UnknownAuthorityError)
return ok
}

View File

@ -0,0 +1,14 @@
// Copyright (c) HashiCorp, Inc.
// SPDX-License-Identifier: MPL-2.0
//go:build go1.20
// +build go1.20
package retryablehttp
import "crypto/tls"
func isCertError(err error) bool {
_, ok := err.(*tls.CertificateVerificationError)
return ok
}

View File

@ -27,10 +27,8 @@ package retryablehttp
import (
"bytes"
"context"
"crypto/x509"
"fmt"
"io"
"io/ioutil"
"log"
"math"
"math/rand"
@ -63,6 +61,10 @@ var (
// limit the size we consume to respReadLimit.
respReadLimit = int64(4096)
// timeNow sets the function that returns the current time.
// This defaults to time.Now. Changes to this should only be done in tests.
timeNow = time.Now
// A regular expression to match the error returned by net/http when the
// configured number of redirects is exhausted. This error isn't typed
// specifically so we resort to matching on the error string.
@ -73,6 +75,11 @@ var (
// specifically so we resort to matching on the error string.
schemeErrorRe = regexp.MustCompile(`unsupported protocol scheme`)
// A regular expression to match the error returned by net/http when a
// request header or value is invalid. This error isn't typed
// specifically so we resort to matching on the error string.
invalidHeaderErrorRe = regexp.MustCompile(`invalid header`)
// A regular expression to match the error returned by net/http when the
// TLS certificate is not trusted. This error isn't typed
// specifically so we resort to matching on the error string.
@ -248,21 +255,19 @@ func getBodyReaderAndContentLength(rawBody interface{}) (ReaderFunc, int64, erro
// deal with it seeking so want it to match here instead of the
// io.ReadSeeker case.
case *bytes.Reader:
buf, err := ioutil.ReadAll(body)
if err != nil {
return nil, 0, err
}
snapshot := *body
bodyReader = func() (io.Reader, error) {
return bytes.NewReader(buf), nil
r := snapshot
return &r, nil
}
contentLength = int64(len(buf))
contentLength = int64(body.Len())
// Compat case
case io.ReadSeeker:
raw := body
bodyReader = func() (io.Reader, error) {
_, err := raw.Seek(0, 0)
return ioutil.NopCloser(raw), err
return io.NopCloser(raw), err
}
if lr, ok := raw.(LenReader); ok {
contentLength = int64(lr.Len())
@ -270,7 +275,7 @@ func getBodyReaderAndContentLength(rawBody interface{}) (ReaderFunc, int64, erro
// Read all in so we can reset
case io.Reader:
buf, err := ioutil.ReadAll(body)
buf, err := io.ReadAll(body)
if err != nil {
return nil, 0, err
}
@ -393,6 +398,9 @@ type Backoff func(min, max time.Duration, attemptNum int, resp *http.Response) t
// attempted. If overriding this, be sure to close the body if needed.
type ErrorHandler func(resp *http.Response, err error, numTries int) (*http.Response, error)
// PrepareRetry is called before retry operation. It can be used for example to re-sign the request
type PrepareRetry func(req *http.Request) error
// Client is used to make HTTP requests. It adds additional functionality
// like automatic retries to tolerate minor outages.
type Client struct {
@ -421,6 +429,9 @@ type Client struct {
// ErrorHandler specifies the custom error handler to use, if any
ErrorHandler ErrorHandler
// PrepareRetry can prepare the request for retry operation, for example re-sign it
PrepareRetry PrepareRetry
loggerInit sync.Once
clientInit sync.Once
}
@ -494,11 +505,16 @@ func baseRetryPolicy(resp *http.Response, err error) (bool, error) {
return false, v
}
// Don't retry if the error was due to an invalid header.
if invalidHeaderErrorRe.MatchString(v.Error()) {
return false, v
}
// Don't retry if the error was due to TLS cert verification failure.
if notTrustedErrorRe.MatchString(v.Error()) {
return false, v
}
if _, ok := v.Err.(x509.UnknownAuthorityError); ok {
if isCertError(v.Err) {
return false, v
}
}
@ -535,10 +551,8 @@ func baseRetryPolicy(resp *http.Response, err error) (bool, error) {
func DefaultBackoff(min, max time.Duration, attemptNum int, resp *http.Response) time.Duration {
if resp != nil {
if resp.StatusCode == http.StatusTooManyRequests || resp.StatusCode == http.StatusServiceUnavailable {
if s, ok := resp.Header["Retry-After"]; ok {
if sleep, err := strconv.ParseInt(s[0], 10, 64); err == nil {
return time.Second * time.Duration(sleep)
}
if sleep, ok := parseRetryAfterHeader(resp.Header["Retry-After"]); ok {
return sleep
}
}
}
@ -551,6 +565,41 @@ func DefaultBackoff(min, max time.Duration, attemptNum int, resp *http.Response)
return sleep
}
// parseRetryAfterHeader parses the Retry-After header and returns the
// delay duration according to the spec: https://httpwg.org/specs/rfc7231.html#header.retry-after
// The bool returned will be true if the header was successfully parsed.
// Otherwise, the header was either not present, or was not parseable according to the spec.
//
// Retry-After headers come in two flavors: Seconds or HTTP-Date
//
// Examples:
// * Retry-After: Fri, 31 Dec 1999 23:59:59 GMT
// * Retry-After: 120
func parseRetryAfterHeader(headers []string) (time.Duration, bool) {
if len(headers) == 0 || headers[0] == "" {
return 0, false
}
header := headers[0]
// Retry-After: 120
if sleep, err := strconv.ParseInt(header, 10, 64); err == nil {
if sleep < 0 { // a negative sleep doesn't make sense
return 0, false
}
return time.Second * time.Duration(sleep), true
}
// Retry-After: Fri, 31 Dec 1999 23:59:59 GMT
retryTime, err := time.Parse(time.RFC1123, header)
if err != nil {
return 0, false
}
if until := retryTime.Sub(timeNow()); until > 0 {
return until, true
}
// date is in the past
return 0, true
}
// LinearJitterBackoff provides a callback for Client.Backoff which will
// perform linear backoff based on the attempt number and with jitter to
// prevent a thundering herd.
@ -578,13 +627,13 @@ func LinearJitterBackoff(min, max time.Duration, attemptNum int, resp *http.Resp
}
// Seed rand; doing this every time is fine
rand := rand.New(rand.NewSource(int64(time.Now().Nanosecond())))
source := rand.New(rand.NewSource(int64(time.Now().Nanosecond())))
// Pick a random number that lies somewhere between the min and max and
// multiply by the attemptNum. attemptNum starts at zero so we always
// increment here. We first get a random percentage, then apply that to the
// difference between min and max, and add to min.
jitter := rand.Float64() * float64(max-min)
jitter := source.Float64() * float64(max-min)
jitterMin := int64(jitter) + int64(min)
return time.Duration(jitterMin * int64(attemptNum))
}
@ -618,10 +667,10 @@ func (c *Client) Do(req *Request) (*http.Response, error) {
var resp *http.Response
var attempt int
var shouldRetry bool
var doErr, respErr, checkErr error
var doErr, respErr, checkErr, prepareErr error
for i := 0; ; i++ {
doErr, respErr = nil, nil
doErr, respErr, prepareErr = nil, nil, nil
attempt++
// Always rewind the request body when non-nil.
@ -634,7 +683,7 @@ func (c *Client) Do(req *Request) (*http.Response, error) {
if c, ok := body.(io.ReadCloser); ok {
req.Body = c
} else {
req.Body = ioutil.NopCloser(body)
req.Body = io.NopCloser(body)
}
}
@ -728,17 +777,26 @@ func (c *Client) Do(req *Request) (*http.Response, error) {
// without racing against the closeBody call in persistConn.writeLoop.
httpreq := *req.Request
req.Request = &httpreq
if c.PrepareRetry != nil {
if err := c.PrepareRetry(req.Request); err != nil {
prepareErr = err
break
}
}
}
// this is the closest we have to success criteria
if doErr == nil && respErr == nil && checkErr == nil && !shouldRetry {
if doErr == nil && respErr == nil && checkErr == nil && prepareErr == nil && !shouldRetry {
return resp, nil
}
defer c.HTTPClient.CloseIdleConnections()
var err error
if checkErr != nil {
if prepareErr != nil {
err = prepareErr
} else if checkErr != nil {
err = checkErr
} else if respErr != nil {
err = respErr
@ -770,7 +828,7 @@ func (c *Client) Do(req *Request) (*http.Response, error) {
// Try to read the response body so we can reuse this connection.
func (c *Client) drainBody(body io.ReadCloser) {
defer body.Close()
_, err := io.Copy(ioutil.Discard, io.LimitReader(body, respReadLimit))
_, err := io.Copy(io.Discard, io.LimitReader(body, respReadLimit))
if err != nil {
if c.logger() != nil {
switch v := c.logger().(type) {

14
vendor/modules.txt vendored
View File

@ -141,8 +141,8 @@ github.com/containernetworking/cni/pkg/version
# github.com/containernetworking/plugins v1.4.1
## explicit; go 1.20
github.com/containernetworking/plugins/pkg/ns
# github.com/containers/buildah v1.35.1-0.20240507152307-453d2fc1099f
## explicit; go 1.20
# github.com/containers/buildah v1.35.1-0.20240510150258-77f239ae12e5
## explicit; go 1.21
github.com/containers/buildah
github.com/containers/buildah/bind
github.com/containers/buildah/chroot
@ -171,7 +171,7 @@ github.com/containers/buildah/pkg/sshagent
github.com/containers/buildah/pkg/util
github.com/containers/buildah/pkg/volumes
github.com/containers/buildah/util
# github.com/containers/common v0.58.1-0.20240508094622-694258d1f523
# github.com/containers/common v0.58.1-0.20240509172903-2c88a3f280bb
## explicit; go 1.21
github.com/containers/common/internal
github.com/containers/common/internal/attributedstring
@ -244,7 +244,7 @@ github.com/containers/conmon/runner/config
# github.com/containers/gvisor-tap-vsock v0.7.4-0.20240408151405-d744d71db363
## explicit; go 1.20
github.com/containers/gvisor-tap-vsock/pkg/types
# github.com/containers/image/v5 v5.30.1-0.20240506211741-f0acff0fe152
# github.com/containers/image/v5 v5.30.2-0.20240509191815-9318d0eaaf78
## explicit; go 1.21
github.com/containers/image/v5/copy
github.com/containers/image/v5/directory
@ -469,7 +469,7 @@ github.com/distribution/reference
github.com/docker/distribution/registry/api/errcode
github.com/docker/distribution/registry/api/v2
github.com/docker/distribution/registry/client/auth/challenge
# github.com/docker/docker v26.1.1+incompatible
# github.com/docker/docker v26.1.2+incompatible
## explicit
github.com/docker/docker/api
github.com/docker/docker/api/types
@ -696,8 +696,8 @@ github.com/hashicorp/go-cleanhttp
# github.com/hashicorp/go-multierror v1.1.1
## explicit; go 1.13
github.com/hashicorp/go-multierror
# github.com/hashicorp/go-retryablehttp v0.7.5
## explicit; go 1.13
# github.com/hashicorp/go-retryablehttp v0.7.6
## explicit; go 1.19
github.com/hashicorp/go-retryablehttp
# github.com/hugelgupf/p9 v0.3.1-0.20230822151754-54f5c5530921
## explicit; go 1.20