opensource/podman
1
0
Fork 0
mirror of https://github.com/containers/podman.git synced 2025-05-20 08:36:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

vendor: update c/common to latest

Browse Source 
Includes some netns cleanup fixes.

Signed-off-by: Paul Holzinger <pholzing@redhat.com>
This commit is contained in:
Paul Holzinger
2024-08-09 13:08:59 +02:00
parent 4aaa5cb6f0
commit be41ee4131
22 changed files with 676 additions and 363 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
go.mod
View File

@ -13,10 +13,10 @@ require (
github.com/checkpoint-restore/go-criu/v7 v7.1.0
github.com/containernetworking/plugins v1.5.1
github.com/containers/buildah v1.37.0
github.com/containers/common v0.60.1-0.20240806133927-05b2e1fd7cd0
github.com/containers/common v0.60.1-0.20240808214705-d93f74f43223
github.com/containers/conmon v2.0.20+incompatible
github.com/containers/gvisor-tap-vsock v0.7.4
github.com/containers/image/v5 v5.32.0
github.com/containers/image/v5 v5.32.1-0.20240806084436-e3e9287ca8e6
github.com/containers/libhvee v0.7.1
github.com/containers/ocicrypt v1.2.0
github.com/containers/psgo v1.9.0
@ -215,7 +215,7 @@ require (
go.opentelemetry.io/otel/trace v1.24.0 // indirect
golang.org/x/arch v0.7.0 // indirect
golang.org/x/mod v0.20.0 // indirect
golang.org/x/oauth2 v0.21.0 // indirect
golang.org/x/oauth2 v0.22.0 // indirect
golang.org/x/time v0.5.0 // indirect
golang.org/x/tools v0.24.0 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 // indirect

12
go.sum
View File

@ -79,14 +79,14 @@ github.com/containernetworking/plugins v1.5.1 h1:T5ji+LPYjjgW0QM+KyrigZbLsZ8jaX+
github.com/containernetworking/plugins v1.5.1/go.mod h1:MIQfgMayGuHYs0XdNudf31cLLAC+i242hNm6KuDGqCM=
github.com/containers/buildah v1.37.0 h1:jvHwu1vIwIqnHyOSg9eef9Apdpry+5oWLrm43gdf8Rk=
github.com/containers/buildah v1.37.0/go.mod h1:MKd79tkluMf6vtH06SedhBQK5OB7E0pFVIuiTTw3dJk=
github.com/containers/common v0.60.1-0.20240806133927-05b2e1fd7cd0 h1:d4oujEs1vClWSRqGEa6OGjh7iKtmuVY8ybrp6JJOt6w=
github.com/containers/common v0.60.1-0.20240806133927-05b2e1fd7cd0/go.mod h1:mWJvbq/MQ8smLKiVwGkcEwzNO+TiFVWA3gpJ2teqgIs=
github.com/containers/common v0.60.1-0.20240808214705-d93f74f43223 h1:/EVrHfr6hmQZw1vDCknv5PvUmFaxYxgkM16S9Iby3+k=
github.com/containers/common v0.60.1-0.20240808214705-d93f74f43223/go.mod h1:qle/jT+ERu905WM/bDkyErUTh9BTdNbbE6d4JVFwT8U=
github.com/containers/conmon v2.0.20+incompatible h1:YbCVSFSCqFjjVwHTPINGdMX1F6JXHGTUje2ZYobNrkg=
github.com/containers/conmon v2.0.20+incompatible/go.mod h1:hgwZ2mtuDrppv78a/cOBNiCm6O0UMWGx1mu7P00nu5I=
github.com/containers/gvisor-tap-vsock v0.7.4 h1:iOtr/KEi+r599OOx1+9Qbss91jD5yxh1HO35MKTdths=
github.com/containers/gvisor-tap-vsock v0.7.4/go.mod h1:orUOSdxU/IGEOxhecu2i7EzV7k7e2TgQlyCBfUngS0A=
github.com/containers/image/v5 v5.32.0 h1:yjbweazPfr8xOzQ2hkkYm1A2V0jN96/kES6Gwyxj7hQ=
github.com/containers/image/v5 v5.32.0/go.mod h1:x5e0RDfGaY6bnQ13gJ2LqbfHvzssfB/y5a8HduGFxJc=
github.com/containers/image/v5 v5.32.1-0.20240806084436-e3e9287ca8e6 h1:nXEEUAo8l2HLlMBy+LsHju2AikpA30jvlTSHbnjJXVw=
github.com/containers/image/v5 v5.32.1-0.20240806084436-e3e9287ca8e6/go.mod h1:r//zsX8SjmVH0F87d+gakcgR4W5HTFGSgSLB4sufW6A=
github.com/containers/libhvee v0.7.1 h1:dWGF5GLq9DZvXo3P8aDp3cNieL5eCaSell4UmeA/jY4=
github.com/containers/libhvee v0.7.1/go.mod h1:fRKB3AyIqHMvq6xaeYhTpckM2cdoq0oecolyoiuLP7M=
github.com/containers/libtrust v0.0.0-20230121012942-c1716e8a8d01 h1:Qzk5C6cYglewc+UyGf6lc8Mj2UaPTHy/iF2De0/77CA=
@ -605,8 +605,8 @@ golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE=
golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs=
golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
golang.org/x/oauth2 v0.22.0 h1:BzDx2FehcG7jJwgWLELCdmLuxk2i+x9UDpSiss2u0ZA=
golang.org/x/oauth2 v0.22.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=

47
vendor/github.com/containers/common/libnetwork/internal/rootlessnetns/netns_linux.go generated vendored
View File

@ -181,7 +181,7 @@ func (n *Netns) cleanup() error {
if err := n.cleanupRootlessNetns(); err != nil {
multiErr = multierror.Append(multiErr, wrapError("kill network process", err))
}
if err := os.RemoveAll(n.dir); err != nil {
if err := os.RemoveAll(n.dir); err != nil && !errors.Is(err, fs.ErrNotExist) {
multiErr = multierror.Append(multiErr, wrapError("remove rootless netns dir", err))
}
@ -280,6 +280,11 @@ func (n *Netns) setupSlirp4netns(nsPath string) error {
func (n *Netns) cleanupRootlessNetns() error {
pidFile := n.getPath(rootlessNetNsConnPidFile)
pid, err := readPidFile(pidFile)
// do not hard error if the file dos not exists, cleanup should be idempotent
if errors.Is(err, fs.ErrNotExist) {
logrus.Debugf("Rootless netns conn pid file does not exists %s", pidFile)
return nil
}
if err == nil {
// kill the slirp/pasta process so we do not leak it
err = unix.Kill(pid, unix.SIGTERM)
@ -512,14 +517,14 @@ func (n *Netns) mountCNIVarDir() error {
return nil
}
func (n *Netns) runInner(toRun func() error) (err error) {
func (n *Netns) runInner(toRun func() error, cleanup bool) (err error) {
nsRef, newNs, err := n.getOrCreateNetns()
if err != nil {
return err
}
defer nsRef.Close()
// If a new netns was created make sure to clean it up again on an error to not leak it.
if newNs {
// If a new netns was created make sure to clean it up again on an error to not leak it if requested.
if newNs && cleanup {
defer func() {
if err != nil {
if err := n.cleanup(); err != nil {
@ -555,7 +560,7 @@ func (n *Netns) runInner(toRun func() error) (err error) {
}
func (n *Netns) Setup(nets int, toRun func() error) error {
err := n.runInner(toRun)
err := n.runInner(toRun, true)
if err != nil {
return err
}
@ -564,25 +569,22 @@ func (n *Netns) Setup(nets int, toRun func() error) error {
}
func (n *Netns) Teardown(nets int, toRun func() error) error {
var multiErr *multierror.Error
count, countErr := refCount(n.dir, -nets)
if countErr != nil {
multiErr = multierror.Append(multiErr, countErr)
}
err := n.runInner(toRun)
err := n.runInner(toRun, true)
if err != nil {
multiErr = multierror.Append(multiErr, err)
return err
}
// decrement only if teardown didn't fail, podman will call us again on errors so we should not double decrement
count, err := refCount(n.dir, -nets)
if err != nil {
return err
}
// only cleanup if the ref count did not throw an error
if count == 0 && countErr == nil {
err = n.cleanup()
if err != nil {
multiErr = multierror.Append(multiErr, wrapError("cleanup", err))
}
// cleanup when ref count is 0
if count == 0 {
return n.cleanup()
}
return multiErr.ErrorOrNil()
return nil
}
// Run any long running function in the userns.
@ -604,7 +606,7 @@ func (n *Netns) Run(lock *lockfile.LockFile, toRun func() error) error {
return err
}
inErr := n.runInner(inner)
inErr := n.runInner(inner, false)
// make sure to always reset the ref counter afterwards
count, err := refCount(n.dir, -1)
if err != nil {
@ -614,9 +616,8 @@ func (n *Netns) Run(lock *lockfile.LockFile, toRun func() error) error {
logrus.Errorf("Failed to decrement ref count: %v", err)
return inErr
}
// runInner() already cleans up the netns when it created a new one on errors
// so we only need to do that if there was no error.
if inErr == nil && count == 0 {
if count == 0 {
err = n.cleanup()
if err != nil {
return wrapError("cleanup", err)

39
vendor/github.com/containers/common/pkg/netns/netns_linux.go generated vendored
View File

@ -28,10 +28,12 @@ import (
"runtime"
"strings"
"sync"
"time"
"github.com/containernetworking/plugins/pkg/ns"
"github.com/containers/storage/pkg/homedir"
"github.com/containers/storage/pkg/unshare"
"github.com/sirupsen/logrus"
"golang.org/x/sys/unix"
)
@ -177,26 +179,35 @@ func newNSPath(nsPath string) (ns.NetNS, error) {
// UnmountNS unmounts the given netns path
func UnmountNS(nsPath string) error {
var rErr error
// Only unmount if it's been bind-mounted (don't touch namespaces in /proc...)
if !strings.HasPrefix(nsPath, "/proc/") {
if err := unix.Unmount(nsPath, unix.MNT_DETACH); err != nil {
// Do not return here, always try to remove below.
// This is important in case podman now is in a new userns compared to
// when the netns was created. The umount will fail EINVAL but removing
// the file will work and the kernel will destroy the bind mount in the
// other ns because of this. We also need it so pasta doesn't leak.
rErr = fmt.Errorf("failed to unmount NS: at %s: %w", nsPath, err)
// EINVAL means the path exists but is not mounted, just try to remove the path below
if err := unix.Unmount(nsPath, unix.MNT_DETACH); err != nil && !errors.Is(err, unix.EINVAL) {
// If path does not exists we can return without error as we have nothing to do.
if errors.Is(err, unix.ENOENT) {
return nil
}
return fmt.Errorf("failed to unmount NS: at %s: %w", nsPath, err)
}
if err := os.Remove(nsPath); err != nil {
err := fmt.Errorf("failed to remove ns path: %w", err)
if rErr != nil {
err = fmt.Errorf("%v, %w", err, rErr)
for {
if err := os.Remove(nsPath); err != nil {
if errors.Is(err, unix.EBUSY) {
// mount is still busy, sleep a moment and try again to remove
logrus.Debugf("Netns %s still busy, try removing it again in 10ms", nsPath)
time.Sleep(10 * time.Millisecond)
continue
}
// If path does not exists we can return without error.
if errors.Is(err, unix.ENOENT) {
break
}
return fmt.Errorf("failed to remove ns path: %w", err)
}
rErr = err
break
}
}
return rErr
return nil
}

121
vendor/github.com/containers/image/v5/copy/compression.go generated vendored
View File

@ -11,6 +11,7 @@ import (
"github.com/containers/image/v5/pkg/compression"
compressiontypes "github.com/containers/image/v5/pkg/compression/types"
"github.com/containers/image/v5/types"
chunkedToc "github.com/containers/storage/pkg/chunked/toc"
imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/sirupsen/logrus"
)
@ -34,10 +35,10 @@ var (
// bpDetectCompressionStepData contains data that the copy pipeline needs about the “detect compression” step.
type bpDetectCompressionStepData struct {
isCompressed bool
format compressiontypes.Algorithm // Valid if isCompressed
decompressor compressiontypes.DecompressorFunc // Valid if isCompressed
srcCompressorName string // Compressor name to possibly record in the blob info cache for the source blob.
isCompressed bool
format compressiontypes.Algorithm // Valid if isCompressed
decompressor compressiontypes.DecompressorFunc // Valid if isCompressed
srcCompressorBaseVariantName string // Compressor name to possibly record in the blob info cache for the source blob.
}
// blobPipelineDetectCompressionStep updates *stream to detect its current compression format.
@ -57,9 +58,9 @@ func blobPipelineDetectCompressionStep(stream *sourceStream, srcInfo types.BlobI
decompressor: decompressor,
}
if res.isCompressed {
res.srcCompressorName = format.Name()
res.srcCompressorBaseVariantName = format.BaseVariantName()
} else {
res.srcCompressorName = internalblobinfocache.Uncompressed
res.srcCompressorBaseVariantName = internalblobinfocache.Uncompressed
}
if expectedBaseFormat, known := expectedBaseCompressionFormats[stream.info.MediaType]; known && res.isCompressed && format.BaseVariantName() != expectedBaseFormat.Name() {
@ -70,13 +71,13 @@ func blobPipelineDetectCompressionStep(stream *sourceStream, srcInfo types.BlobI
// bpCompressionStepData contains data that the copy pipeline needs about the compression step.
type bpCompressionStepData struct {
operation bpcOperation // What we are actually doing
uploadedOperation types.LayerCompression // Operation to use for updating the blob metadata (matching the end state, not necessarily what we do)
uploadedAlgorithm *compressiontypes.Algorithm // An algorithm parameter for the compressionOperation edits.
uploadedAnnotations map[string]string // Compression-related annotations that should be set on the uploaded blob. WARNING: This is only set after the srcStream.reader is fully consumed.
srcCompressorName string // Compressor name to record in the blob info cache for the source blob.
uploadedCompressorName string // Compressor name to record in the blob info cache for the uploaded blob.
closers []io.Closer // Objects to close after the upload is done, if any.
operation bpcOperation // What we are actually doing
uploadedOperation types.LayerCompression // Operation to use for updating the blob metadata (matching the end state, not necessarily what we do)
uploadedAlgorithm *compressiontypes.Algorithm // An algorithm parameter for the compressionOperation edits.
uploadedAnnotations map[string]string // Compression-related annotations that should be set on the uploaded blob. WARNING: This is only set after the srcStream.reader is fully consumed.
srcCompressorBaseVariantName string // Compressor base variant name to record in the blob info cache for the source blob.
uploadedCompressorName string // Compressor name to record in the blob info cache for the uploaded blob.
closers []io.Closer // Objects to close after the upload is done, if any.
}
type bpcOperation int
@ -128,11 +129,11 @@ func (ic *imageCopier) bpcPreserveEncrypted(stream *sourceStream, _ bpDetectComp
// We can’t do anything with an encrypted blob unless decrypted.
logrus.Debugf("Using original blob without modification for encrypted blob")
return &bpCompressionStepData{
operation: bpcOpPreserveOpaque,
uploadedOperation: types.PreserveOriginal,
uploadedAlgorithm: nil,
srcCompressorName: internalblobinfocache.UnknownCompression,
uploadedCompressorName: internalblobinfocache.UnknownCompression,
operation: bpcOpPreserveOpaque,
uploadedOperation: types.PreserveOriginal,
uploadedAlgorithm: nil,
srcCompressorBaseVariantName: internalblobinfocache.UnknownCompression,
uploadedCompressorName: internalblobinfocache.UnknownCompression,
}, nil
}
return nil, nil
@ -157,13 +158,13 @@ func (ic *imageCopier) bpcCompressUncompressed(stream *sourceStream, detected bp
Size: -1,
}
return &bpCompressionStepData{
operation: bpcOpCompressUncompressed,
uploadedOperation: types.Compress,
uploadedAlgorithm: uploadedAlgorithm,
uploadedAnnotations: annotations,
srcCompressorName: detected.srcCompressorName,
uploadedCompressorName: uploadedAlgorithm.Name(),
closers: []io.Closer{reader},
operation: bpcOpCompressUncompressed,
uploadedOperation: types.Compress,
uploadedAlgorithm: uploadedAlgorithm,
uploadedAnnotations: annotations,
srcCompressorBaseVariantName: detected.srcCompressorBaseVariantName,
uploadedCompressorName: uploadedAlgorithm.Name(),
closers: []io.Closer{reader},
}, nil
}
return nil, nil
@ -198,13 +199,13 @@ func (ic *imageCopier) bpcRecompressCompressed(stream *sourceStream, detected bp
}
succeeded = true
return &bpCompressionStepData{
operation: bpcOpRecompressCompressed,
uploadedOperation: types.PreserveOriginal,
uploadedAlgorithm: ic.compressionFormat,
uploadedAnnotations: annotations,
srcCompressorName: detected.srcCompressorName,
uploadedCompressorName: ic.compressionFormat.Name(),
closers: []io.Closer{decompressed, recompressed},
operation: bpcOpRecompressCompressed,
uploadedOperation: types.PreserveOriginal,
uploadedAlgorithm: ic.compressionFormat,
uploadedAnnotations: annotations,
srcCompressorBaseVariantName: detected.srcCompressorBaseVariantName,
uploadedCompressorName: ic.compressionFormat.Name(),
closers: []io.Closer{decompressed, recompressed},
}, nil
}
return nil, nil
@ -225,12 +226,12 @@ func (ic *imageCopier) bpcDecompressCompressed(stream *sourceStream, detected bp
Size: -1,
}
return &bpCompressionStepData{
operation: bpcOpDecompressCompressed,
uploadedOperation: types.Decompress,
uploadedAlgorithm: nil,
srcCompressorName: detected.srcCompressorName,
uploadedCompressorName: internalblobinfocache.Uncompressed,
closers: []io.Closer{s},
operation: bpcOpDecompressCompressed,
uploadedOperation: types.Decompress,
uploadedAlgorithm: nil,
srcCompressorBaseVariantName: detected.srcCompressorBaseVariantName,
uploadedCompressorName: internalblobinfocache.Uncompressed,
closers: []io.Closer{s},
}, nil
}
return nil, nil
@ -268,11 +269,14 @@ func (ic *imageCopier) bpcPreserveOriginal(_ *sourceStream, detected bpDetectCom
algorithm = nil
}
return &bpCompressionStepData{
operation: bpcOp,
uploadedOperation: uploadedOp,
uploadedAlgorithm: algorithm,
srcCompressorName: detected.srcCompressorName,
uploadedCompressorName: detected.srcCompressorName,
operation: bpcOp,
uploadedOperation: uploadedOp,
uploadedAlgorithm: algorithm,
srcCompressorBaseVariantName: detected.srcCompressorBaseVariantName,
// We only record the base variant of the format on upload; we didn’t do anything with
// the TOC, we don’t know whether it matches the blob digest, so we don’t want to trigger
// reuse of any kind between the blob digest and the TOC digest.
uploadedCompressorName: detected.srcCompressorBaseVariantName,
}
}
@ -308,6 +312,15 @@ func (d *bpCompressionStepData) recordValidatedDigestData(c *copier, uploadedInf
// No useful information
case bpcOpCompressUncompressed:
c.blobInfoCache.RecordDigestUncompressedPair(uploadedInfo.Digest, srcInfo.Digest)
if d.uploadedAnnotations != nil {
tocDigest, err := chunkedToc.GetTOCDigest(d.uploadedAnnotations)
if err != nil {
return fmt.Errorf("parsing just-created compression annotations: %w", err)
}
if tocDigest != nil {
c.blobInfoCache.RecordTOCUncompressedPair(*tocDigest, srcInfo.Digest)
}
}
case bpcOpDecompressCompressed:
c.blobInfoCache.RecordDigestUncompressedPair(srcInfo.Digest, uploadedInfo.Digest)
case bpcOpRecompressCompressed, bpcOpPreserveCompressed:
@ -323,9 +336,9 @@ func (d *bpCompressionStepData) recordValidatedDigestData(c *copier, uploadedInf
return fmt.Errorf("Internal error: Unexpected d.operation value %#v", d.operation)
}
}
if d.srcCompressorName == "" || d.uploadedCompressorName == "" {
return fmt.Errorf("internal error: missing compressor names (src: %q, uploaded: %q)",
d.srcCompressorName, d.uploadedCompressorName)
if d.srcCompressorBaseVariantName == "" || d.uploadedCompressorName == "" {
return fmt.Errorf("internal error: missing compressor names (src base: %q, uploaded: %q)",
d.srcCompressorBaseVariantName, d.uploadedCompressorName)
}
if d.uploadedCompressorName != internalblobinfocache.UnknownCompression {
if d.uploadedCompressorName != compressiontypes.ZstdChunkedAlgorithmName {
@ -337,15 +350,19 @@ func (d *bpCompressionStepData) recordValidatedDigestData(c *copier, uploadedInf
// between zstd and zstd:chunked; so we could, in varying situations over time, call RecordDigestCompressorName
// with the same digest and both ZstdAlgorithmName and ZstdChunkedAlgorithmName , which causes warnings about
// inconsistent data to be logged.
c.blobInfoCache.RecordDigestCompressorName(uploadedInfo.Digest, d.uploadedCompressorName)
c.blobInfoCache.RecordDigestCompressorData(uploadedInfo.Digest, internalblobinfocache.DigestCompressorData{
BaseVariantCompressor: d.uploadedCompressorName,
})
}
}
if srcInfo.Digest != "" && srcInfo.Digest != uploadedInfo.Digest &&
d.srcCompressorName != internalblobinfocache.UnknownCompression {
if d.srcCompressorName != compressiontypes.ZstdChunkedAlgorithmName {
// HACK: Don’t record zstd:chunked algorithms, see above.
c.blobInfoCache.RecordDigestCompressorName(srcInfo.Digest, d.srcCompressorName)
}
d.srcCompressorBaseVariantName != internalblobinfocache.UnknownCompression {
// If the source is already using some TOC-dependent variant, we either copied the
// blob as is, or perhaps decompressed it; either way we don’t trust the TOC digest,
// so record neither the variant name, nor the TOC digest.
c.blobInfoCache.RecordDigestCompressorData(srcInfo.Digest, internalblobinfocache.DigestCompressorData{
BaseVariantCompressor: d.srcCompressorBaseVariantName,
})
}
return nil
}

2
vendor/github.com/containers/image/v5/copy/encryption.go generated vendored
View File

@ -48,7 +48,7 @@ func (ic *imageCopier) blobPipelineDecryptionStep(stream *sourceStream, srcInfo
Annotations: stream.info.Annotations,
}
// DecryptLayer supposedly returns a digest of the decrypted stream.
// In pratice, that value is never set in the current implementation.
// In practice, that value is never set in the current implementation.
// And we shouldn’t use it anyway, because it is not trusted: encryption can be made to a public key,
// i.e. it doesn’t authenticate the origin of the metadata in any way.
reader, _, err := ocicrypt.DecryptLayer(ic.c.options.OciDecryptConfig, stream.reader, desc, false)

2
vendor/github.com/containers/image/v5/copy/progress_bars.go generated vendored
View File

@ -121,7 +121,7 @@ func (c *copier) printCopyInfo(kind string, info types.BlobInfo) {
}
}
// mark100PercentComplete marks the progres bars as 100% complete;
// mark100PercentComplete marks the progress bars as 100% complete;
// it may do so by possibly advancing the current state if it is below the known total.
func (bar *progressBar) mark100PercentComplete() {
if bar.originalSize > 0 {

24
vendor/github.com/containers/image/v5/copy/single.go generated vendored
View File

@ -149,6 +149,28 @@ func (c *copier) copySingleImage(ctx context.Context, unparsedImage *image.Unpar
ic.compressionFormat = c.options.DestinationCtx.CompressionFormat
ic.compressionLevel = c.options.DestinationCtx.CompressionLevel
}
// HACK: Don’t combine zstd:chunked and encryption.
// zstd:chunked can only usefully be consumed using range requests of parts of the layer, which would require the encryption
// to support decrypting arbitrary subsets of the stream. That’s plausible but not supported using the encryption API we have.
// Also, the chunked metadata is exposed in annotations unencrypted, which reveals the TOC digest = layer identity without
// encryption. (That can be determined from the unencrypted config anyway, but, still...)
//
// Ideally this should query a well-defined property of the compression algorithm (and $somehow determine the right fallback) instead of
// hard-coding zstd:chunked / zstd.
if ic.c.options.OciEncryptLayers != nil {
format := ic.compressionFormat
if format == nil {
format = defaultCompressionFormat
}
if format.Name() == compression.ZstdChunked.Name() {
if ic.requireCompressionFormatMatch {
return copySingleImageResult{}, errors.New("explicitly requested to combine zstd:chunked with encryption, which is not beneficial; use plain zstd instead")
}
logrus.Warnf("Compression using zstd:chunked is not beneficial for encrypted layers, using plain zstd instead")
ic.compressionFormat = &compression.Zstd
}
}
// Decide whether we can substitute blobs with semantic equivalents:
// - Don’t do that if we can’t modify the manifest at all
// - Ensure _this_ copy sees exactly the intended data when either processing a signed image or signing it.
@ -192,7 +214,7 @@ func (c *copier) copySingleImage(ctx context.Context, unparsedImage *image.Unpar
shouldUpdateSigs := len(sigs) > 0 || len(c.signers) != 0 // TODO: Consider allowing signatures updates only and skipping the image's layers/manifest copy if possible
noPendingManifestUpdates := ic.noPendingManifestUpdates()
logrus.Debugf("Checking if we can skip copying: has signatures=%t, OCI encryption=%t, no manifest updates=%t, compression match required for resuing blobs=%t", shouldUpdateSigs, destRequiresOciEncryption, noPendingManifestUpdates, opts.requireCompressionFormatMatch)
logrus.Debugf("Checking if we can skip copying: has signatures=%t, OCI encryption=%t, no manifest updates=%t, compression match required for reusing blobs=%t", shouldUpdateSigs, destRequiresOciEncryption, noPendingManifestUpdates, opts.requireCompressionFormatMatch)
if !shouldUpdateSigs && !destRequiresOciEncryption && noPendingManifestUpdates && !ic.requireCompressionFormatMatch {
matchedResult, err := ic.compareImageDestinationManifestEqual(ctx, targetInstance)
if err != nil {

9
vendor/github.com/containers/image/v5/internal/blobinfocache/blobinfocache.go generated vendored
View File

@ -27,7 +27,14 @@ func (bic *v1OnlyBlobInfoCache) Open() {
func (bic *v1OnlyBlobInfoCache) Close() {
}
func (bic *v1OnlyBlobInfoCache) RecordDigestCompressorName(anyDigest digest.Digest, compressorName string) {
func (bic *v1OnlyBlobInfoCache) UncompressedDigestForTOC(tocDigest digest.Digest) digest.Digest {
return ""
}
func (bic *v1OnlyBlobInfoCache) RecordTOCUncompressedPair(tocDigest digest.Digest, uncompressed digest.Digest) {
}
func (bic *v1OnlyBlobInfoCache) RecordDigestCompressorData(anyDigest digest.Digest, data DigestCompressorData) {
}
func (bic *v1OnlyBlobInfoCache) CandidateLocations2(transport types.ImageTransport, scope types.BICTransportScope, digest digest.Digest, options CandidateLocations2Options) []BICReplacementCandidate2 {

25
vendor/github.com/containers/image/v5/internal/blobinfocache/types.go generated vendored
View File

@ -26,19 +26,34 @@ type BlobInfoCache2 interface {
// Close destroys state created by Open().
Close()
// RecordDigestCompressorName records a compressor for the blob with the specified digest,
// or Uncompressed or UnknownCompression.
// WARNING: Only call this with LOCALLY VERIFIED data; don’t record a compressor for a
// digest just because some remote author claims so (e.g. because a manifest says so);
// UncompressedDigestForTOC returns an uncompressed digest corresponding to anyDigest.
// Returns "" if the uncompressed digest is unknown.
UncompressedDigestForTOC(tocDigest digest.Digest) digest.Digest
// RecordTOCUncompressedPair records that the tocDigest corresponds to uncompressed.
// WARNING: Only call this for LOCALLY VERIFIED data; don’t record a digest pair just because some remote author claims so (e.g.
// because a manifest/config pair exists); otherwise the cache could be poisoned and allow substituting unexpected blobs.
// (Eventually, the DiffIDs in image config could detect the substitution, but that may be too late, and not all image formats contain that data.)
RecordTOCUncompressedPair(tocDigest digest.Digest, uncompressed digest.Digest)
// RecordDigestCompressorData records data for the blob with the specified digest.
// WARNING: Only call this with LOCALLY VERIFIED data:
// - don’t record a compressor for a digest just because some remote author claims so
// (e.g. because a manifest says so);
// otherwise the cache could be poisoned and cause us to make incorrect edits to type
// information in a manifest.
RecordDigestCompressorName(anyDigest digest.Digest, compressorName string)
RecordDigestCompressorData(anyDigest digest.Digest, data DigestCompressorData)
// CandidateLocations2 returns a prioritized, limited, number of blobs and their locations (if known)
// that could possibly be reused within the specified (transport scope) (if they still
// exist, which is not guaranteed).
CandidateLocations2(transport types.ImageTransport, scope types.BICTransportScope, digest digest.Digest, options CandidateLocations2Options) []BICReplacementCandidate2
}
// DigestCompressorData is information known about how a blob is compressed.
// (This is worded generically, but basically targeted at the zstd / zstd:chunked situation.)
type DigestCompressorData struct {
BaseVariantCompressor string // A compressor’s base variant name, or Uncompressed or UnknownCompression.
}
// CandidateLocations2Options are used in CandidateLocations2.
type CandidateLocations2Options struct {
// If !CanSubstitute, the returned candidates will match the submitted digest exactly; if

2
vendor/github.com/containers/image/v5/manifest/docker_schema2.go generated vendored
View File

@ -202,7 +202,7 @@ func (m *Schema2) ConfigInfo() types.BlobInfo {
// The Digest field is guaranteed to be provided; Size may be -1.
// WARNING: The list may contain duplicates, and they are semantically relevant.
func (m *Schema2) LayerInfos() []LayerInfo {
blobs := []LayerInfo{}
blobs := make([]LayerInfo, 0, len(m.LayersDescriptors))
for _, layer := range m.LayersDescriptors {
blobs = append(blobs, LayerInfo{
BlobInfo: BlobInfoFromSchema2Descriptor(layer),

2
vendor/github.com/containers/image/v5/manifest/oci.go generated vendored
View File

@ -95,7 +95,7 @@ func (m *OCI1) ConfigInfo() types.BlobInfo {
// The Digest field is guaranteed to be provided; Size may be -1.
// WARNING: The list may contain duplicates, and they are semantically relevant.
func (m *OCI1) LayerInfos() []LayerInfo {
blobs := []LayerInfo{}
blobs := make([]LayerInfo, 0, len(m.Layers))
for _, layer := range m.Layers {
blobs = append(blobs, LayerInfo{
BlobInfo: BlobInfoFromOCI1Descriptor(layer),

6
vendor/github.com/containers/image/v5/ostree/ostree_src.go generated vendored
View File

@ -151,9 +151,9 @@ func openRepo(path string) (*C.struct_OstreeRepo, error) {
var cerr *C.GError
cpath := C.CString(path)
defer C.free(unsafe.Pointer(cpath))
pathc := C.g_file_new_for_path(cpath)
defer C.g_object_unref(C.gpointer(pathc))
repo := C.ostree_repo_new(pathc)
file := C.g_file_new_for_path(cpath)
defer C.g_object_unref(C.gpointer(file))
repo := C.ostree_repo_new(file)
r := glib.GoBool(glib.GBoolean(C.ostree_repo_open(repo, nil, &cerr)))
if !r {
C.g_object_unref(C.gpointer(repo))

134
vendor/github.com/containers/image/v5/pkg/blobinfocache/internal/prioritize/prioritize.go generated vendored
View File

@ -25,57 +25,103 @@ const replacementAttempts = 5
// This is a heuristic/guess, and could well use a different value.
const replacementUnknownLocationAttempts = 2
// CandidateCompression returns (true, compressionOp, compressionAlgo) if a blob
// with compressionName (which can be Uncompressed or UnknownCompression) is acceptable for a CandidateLocations* call with v2Options.
// CandidateTemplate is a subset of BICReplacementCandidate2 with data related to a specific digest,
// which can be later combined with information about a location.
type CandidateTemplate struct {
digest digest.Digest
compressionOperation types.LayerCompression // Either types.Decompress for uncompressed, or types.Compress for compressed
compressionAlgorithm *compression.Algorithm // An algorithm when the candidate is compressed, or nil when it is uncompressed
}
// CandidateTemplateWithCompression returns a CandidateTemplate if a blob with data is acceptable
// for a CandidateLocations* call with v2Options.
//
// v2Options can be set to nil if the call is CandidateLocations (i.e. compression is not required to be known);
// if not nil, the call is assumed to be CandidateLocations2.
//
// The (compressionOp, compressionAlgo) values are suitable for BICReplacementCandidate2
func CandidateCompression(v2Options *blobinfocache.CandidateLocations2Options, digest digest.Digest, compressorName string) (bool, types.LayerCompression, *compression.Algorithm) {
func CandidateTemplateWithCompression(v2Options *blobinfocache.CandidateLocations2Options, digest digest.Digest, data blobinfocache.DigestCompressorData) *CandidateTemplate {
if v2Options == nil {
return true, types.PreserveOriginal, nil // Anything goes. The (compressionOp, compressionAlgo) values are not used.
return &CandidateTemplate{ // Anything goes. The compressionOperation, compressionAlgorithm values are not used.
digest: digest,
}
}
var op types.LayerCompression
var algo *compression.Algorithm
switch compressorName {
requiredCompression := "nil"
if v2Options.RequiredCompression != nil {
requiredCompression = v2Options.RequiredCompression.Name()
}
switch data.BaseVariantCompressor {
case blobinfocache.Uncompressed:
op = types.Decompress
algo = nil
if !manifest.CandidateCompressionMatchesReuseConditions(manifest.ReuseConditions{
PossibleManifestFormats: v2Options.PossibleManifestFormats,
RequiredCompression: v2Options.RequiredCompression,
}, nil) {
logrus.Debugf("Ignoring BlobInfoCache record of digest %q, uncompressed format does not match required %s or MIME types %#v",
digest.String(), requiredCompression, v2Options.PossibleManifestFormats)
return nil
}
return &CandidateTemplate{
digest: digest,
compressionOperation: types.Decompress,
compressionAlgorithm: nil,
}
case blobinfocache.UnknownCompression:
logrus.Debugf("Ignoring BlobInfoCache record of digest %q with unknown compression", digest.String())
return false, types.PreserveOriginal, nil // Not allowed with CandidateLocations2
return nil // Not allowed with CandidateLocations2
default:
op = types.Compress
algo_, err := compression.AlgorithmByName(compressorName)
algo, err := compression.AlgorithmByName(data.BaseVariantCompressor)
if err != nil {
logrus.Debugf("Ignoring BlobInfoCache record of digest %q with unrecognized compression %q: %v",
digest.String(), compressorName, err)
return false, types.PreserveOriginal, nil // The BICReplacementCandidate2.CompressionAlgorithm field is required
digest.String(), data.BaseVariantCompressor, err)
return nil // The BICReplacementCandidate2.CompressionAlgorithm field is required
}
algo = &algo_
}
if !manifest.CandidateCompressionMatchesReuseConditions(manifest.ReuseConditions{
PossibleManifestFormats: v2Options.PossibleManifestFormats,
RequiredCompression: v2Options.RequiredCompression,
}, algo) {
requiredCompresssion := "nil"
if v2Options.RequiredCompression != nil {
requiredCompresssion = v2Options.RequiredCompression.Name()
if !manifest.CandidateCompressionMatchesReuseConditions(manifest.ReuseConditions{
PossibleManifestFormats: v2Options.PossibleManifestFormats,
RequiredCompression: v2Options.RequiredCompression,
}, &algo) {
logrus.Debugf("Ignoring BlobInfoCache record of digest %q, compression %q does not match required %s or MIME types %#v",
digest.String(), data.BaseVariantCompressor, requiredCompression, v2Options.PossibleManifestFormats)
return nil
}
return &CandidateTemplate{
digest: digest,
compressionOperation: types.Compress,
compressionAlgorithm: &algo,
}
logrus.Debugf("Ignoring BlobInfoCache record of digest %q, compression %q does not match required %s or MIME types %#v",
digest.String(), compressorName, requiredCompresssion, v2Options.PossibleManifestFormats)
return false, types.PreserveOriginal, nil
}
return true, op, algo
}
// CandidateWithTime is the input to types.BICReplacementCandidate prioritization.
type CandidateWithTime struct {
Candidate blobinfocache.BICReplacementCandidate2 // The replacement candidate
LastSeen time.Time // Time the candidate was last known to exist (either read or written) (not set for Candidate.UnknownLocation)
candidate blobinfocache.BICReplacementCandidate2 // The replacement candidate
lastSeen time.Time // Time the candidate was last known to exist (either read or written) (not set for Candidate.UnknownLocation)
}
// CandidateWithLocation returns a complete CandidateWithTime combining (template from CandidateTemplateWithCompression, location, lastSeen)
func (template CandidateTemplate) CandidateWithLocation(location types.BICLocationReference, lastSeen time.Time) CandidateWithTime {
return CandidateWithTime{
candidate: blobinfocache.BICReplacementCandidate2{
Digest: template.digest,
CompressionOperation: template.compressionOperation,
CompressionAlgorithm: template.compressionAlgorithm,
UnknownLocation: false,
Location: location,
},
lastSeen: lastSeen,
}
}
// CandidateWithUnknownLocation returns a complete CandidateWithTime for a template from CandidateTemplateWithCompression and an unknown location.
func (template CandidateTemplate) CandidateWithUnknownLocation() CandidateWithTime {
return CandidateWithTime{
candidate: blobinfocache.BICReplacementCandidate2{
Digest: template.digest,
CompressionOperation: template.compressionOperation,
CompressionAlgorithm: template.compressionAlgorithm,
UnknownLocation: true,
Location: types.BICLocationReference{Opaque: ""},
},
lastSeen: time.Time{},
}
}
// candidateSortState is a closure for a comparison used by slices.SortFunc on candidates to prioritize,
@ -91,35 +137,35 @@ func (css *candidateSortState) compare(xi, xj CandidateWithTime) int {
// Other digest values are primarily sorted by time (more recent first), secondarily by digest (to provide a deterministic order)
// First, deal with the primaryDigest/uncompressedDigest cases:
if xi.Candidate.Digest != xj.Candidate.Digest {
if xi.candidate.Digest != xj.candidate.Digest {
// - The two digests are different, and one (or both) of the digests is primaryDigest or uncompressedDigest: time does not matter
if xi.Candidate.Digest == css.primaryDigest {
if xi.candidate.Digest == css.primaryDigest {
return -1
}
if xj.Candidate.Digest == css.primaryDigest {
if xj.candidate.Digest == css.primaryDigest {
return 1
}
if css.uncompressedDigest != "" {
if xi.Candidate.Digest == css.uncompressedDigest {
if xi.candidate.Digest == css.uncompressedDigest {
return 1
}
if xj.Candidate.Digest == css.uncompressedDigest {
if xj.candidate.Digest == css.uncompressedDigest {
return -1
}
}
} else { // xi.Candidate.Digest == xj.Candidate.Digest
// The two digests are the same, and are either primaryDigest or uncompressedDigest: order by time
if xi.Candidate.Digest == css.primaryDigest || (css.uncompressedDigest != "" && xi.Candidate.Digest == css.uncompressedDigest) {
return -xi.LastSeen.Compare(xj.LastSeen)
if xi.candidate.Digest == css.primaryDigest || (css.uncompressedDigest != "" && xi.candidate.Digest == css.uncompressedDigest) {
return -xi.lastSeen.Compare(xj.lastSeen)
}
}
// Neither of the digests are primaryDigest/uncompressedDigest:
if cmp := xi.LastSeen.Compare(xj.LastSeen); cmp != 0 { // Order primarily by time
if cmp := xi.lastSeen.Compare(xj.lastSeen); cmp != 0 { // Order primarily by time
return -cmp
}
// Fall back to digest, if timestamps end up _exactly_ the same (how?!)
return cmp.Compare(xi.Candidate.Digest, xj.Candidate.Digest)
return cmp.Compare(xi.candidate.Digest, xj.candidate.Digest)
}
// destructivelyPrioritizeReplacementCandidatesWithMax is destructivelyPrioritizeReplacementCandidates with parameters for the
@ -138,7 +184,7 @@ func destructivelyPrioritizeReplacementCandidatesWithMax(cs []CandidateWithTime,
uncompressedDigest: uncompressedDigest,
}).compare)
for _, candidate := range cs {
if candidate.Candidate.UnknownLocation {
if candidate.candidate.UnknownLocation {
unknownLocationCandidates = append(unknownLocationCandidates, candidate)
} else {
knownLocationCandidates = append(knownLocationCandidates, candidate)
@ -150,11 +196,11 @@ func destructivelyPrioritizeReplacementCandidatesWithMax(cs []CandidateWithTime,
unknownLocationCandidatesUsed := min(noLocationLimit, remainingCapacity, len(unknownLocationCandidates))
res := make([]blobinfocache.BICReplacementCandidate2, knownLocationCandidatesUsed)
for i := 0; i < knownLocationCandidatesUsed; i++ {
res[i] = knownLocationCandidates[i].Candidate
res[i] = knownLocationCandidates[i].candidate
}
// If candidates with unknown location are found, lets add them to final list
for i := 0; i < unknownLocationCandidatesUsed; i++ {
res = append(res, unknownLocationCandidates[i].Candidate)
res = append(res, unknownLocationCandidates[i].candidate)
}
return res
}

90
vendor/github.com/containers/image/v5/pkg/blobinfocache/memory/memory.go generated vendored
View File

@ -24,10 +24,11 @@ type locationKey struct {
type cache struct {
mutex sync.Mutex
// The following fields can only be accessed with mutex held.
uncompressedDigests map[digest.Digest]digest.Digest
digestsByUncompressed map[digest.Digest]*set.Set[digest.Digest] // stores a set of digests for each uncompressed digest
knownLocations map[locationKey]map[types.BICLocationReference]time.Time // stores last known existence time for each location reference
compressors map[digest.Digest]string // stores a compressor name, or blobinfocache.Uncompressed (not blobinfocache.UnknownCompression), for each digest
uncompressedDigests map[digest.Digest]digest.Digest
uncompressedDigestsByTOC map[digest.Digest]digest.Digest
digestsByUncompressed map[digest.Digest]*set.Set[digest.Digest] // stores a set of digests for each uncompressed digest
knownLocations map[locationKey]map[types.BICLocationReference]time.Time // stores last known existence time for each location reference
compressors map[digest.Digest]string // stores a compressor name, or blobinfocache.Uncompressed (not blobinfocache.UnknownCompression), for each digest
}
// New returns a BlobInfoCache implementation which is in-memory only.
@ -44,10 +45,11 @@ func New() types.BlobInfoCache {
func new2() *cache {
return &cache{
uncompressedDigests: map[digest.Digest]digest.Digest{},
digestsByUncompressed: map[digest.Digest]*set.Set[digest.Digest]{},
knownLocations: map[locationKey]map[types.BICLocationReference]time.Time{},
compressors: map[digest.Digest]string{},
uncompressedDigests: map[digest.Digest]digest.Digest{},
uncompressedDigestsByTOC: map[digest.Digest]digest.Digest{},
digestsByUncompressed: map[digest.Digest]*set.Set[digest.Digest]{},
knownLocations: map[locationKey]map[types.BICLocationReference]time.Time{},
compressors: map[digest.Digest]string{},
}
}
@ -104,6 +106,30 @@ func (mem *cache) RecordDigestUncompressedPair(anyDigest digest.Digest, uncompre
anyDigestSet.Add(anyDigest)
}
// UncompressedDigestForTOC returns an uncompressed digest corresponding to anyDigest.
// Returns "" if the uncompressed digest is unknown.
func (mem *cache) UncompressedDigestForTOC(tocDigest digest.Digest) digest.Digest {
mem.mutex.Lock()
defer mem.mutex.Unlock()
if d, ok := mem.uncompressedDigestsByTOC[tocDigest]; ok {
return d
}
return ""
}
// RecordTOCUncompressedPair records that the tocDigest corresponds to uncompressed.
// WARNING: Only call this for LOCALLY VERIFIED data; don’t record a digest pair just because some remote author claims so (e.g.
// because a manifest/config pair exists); otherwise the cache could be poisoned and allow substituting unexpected blobs.
// (Eventually, the DiffIDs in image config could detect the substitution, but that may be too late, and not all image formats contain that data.)
func (mem *cache) RecordTOCUncompressedPair(tocDigest digest.Digest, uncompressed digest.Digest) {
mem.mutex.Lock()
defer mem.mutex.Unlock()
if previous, ok := mem.uncompressedDigestsByTOC[tocDigest]; ok && previous != uncompressed {
logrus.Warnf("Uncompressed digest for blob with TOC %q previously recorded as %q, now %q", tocDigest, previous, uncompressed)
}
mem.uncompressedDigestsByTOC[tocDigest] = uncompressed
}
// RecordKnownLocation records that a blob with the specified digest exists within the specified (transport, scope) scope,
// and can be reused given the opaque location data.
func (mem *cache) RecordKnownLocation(transport types.ImageTransport, scope types.BICTransportScope, blobDigest digest.Digest, location types.BICLocationReference) {
@ -118,19 +144,24 @@ func (mem *cache) RecordKnownLocation(transport types.ImageTransport, scope type
locationScope[location] = time.Now() // Possibly overwriting an older entry.
}
// RecordDigestCompressorName records that the blob with the specified digest is either compressed with the specified
// algorithm, or uncompressed, or that we no longer know.
func (mem *cache) RecordDigestCompressorName(blobDigest digest.Digest, compressorName string) {
// RecordDigestCompressorData records data for the blob with the specified digest.
// WARNING: Only call this with LOCALLY VERIFIED data:
// - don’t record a compressor for a digest just because some remote author claims so
// (e.g. because a manifest says so);
//
// otherwise the cache could be poisoned and cause us to make incorrect edits to type
// information in a manifest.
func (mem *cache) RecordDigestCompressorData(anyDigest digest.Digest, data blobinfocache.DigestCompressorData) {
mem.mutex.Lock()
defer mem.mutex.Unlock()
if previous, ok := mem.compressors[blobDigest]; ok && previous != compressorName {
logrus.Warnf("Compressor for blob with digest %s previously recorded as %s, now %s", blobDigest, previous, compressorName)
if previous, ok := mem.compressors[anyDigest]; ok && previous != data.BaseVariantCompressor {
logrus.Warnf("Compressor for blob with digest %s previously recorded as %s, now %s", anyDigest, previous, data.BaseVariantCompressor)
}
if compressorName == blobinfocache.UnknownCompression {
delete(mem.compressors, blobDigest)
if data.BaseVariantCompressor == blobinfocache.UnknownCompression {
delete(mem.compressors, anyDigest)
return
}
mem.compressors[blobDigest] = compressorName
mem.compressors[anyDigest] = data.BaseVariantCompressor
}
// appendReplacementCandidates creates prioritize.CandidateWithTime values for digest in memory
@ -144,34 +175,19 @@ func (mem *cache) appendReplacementCandidates(candidates []prioritize.CandidateW
if v, ok := mem.compressors[digest]; ok {
compressorName = v
}
ok, compressionOp, compressionAlgo := prioritize.CandidateCompression(v2Options, digest, compressorName)
if !ok {
template := prioritize.CandidateTemplateWithCompression(v2Options, digest, blobinfocache.DigestCompressorData{
BaseVariantCompressor: compressorName,
})
if template == nil {
return candidates
}
locations := mem.knownLocations[locationKey{transport: transport.Name(), scope: scope, blobDigest: digest}] // nil if not present
if len(locations) > 0 {
for l, t := range locations {
candidates = append(candidates, prioritize.CandidateWithTime{
Candidate: blobinfocache.BICReplacementCandidate2{
Digest: digest,
CompressionOperation: compressionOp,
CompressionAlgorithm: compressionAlgo,
Location: l,
},
LastSeen: t,
})
candidates = append(candidates, template.CandidateWithLocation(l, t))
}
} else if v2Options != nil {
candidates = append(candidates, prioritize.CandidateWithTime{
Candidate: blobinfocache.BICReplacementCandidate2{
Digest: digest,
CompressionOperation: compressionOp,
CompressionAlgorithm: compressionAlgo,
UnknownLocation: true,
Location: types.BICLocationReference{Opaque: ""},
},
LastSeen: time.Time{},
})
candidates = append(candidates, template.CandidateWithUnknownLocation())
}
return candidates
}

13
vendor/github.com/containers/image/v5/pkg/blobinfocache/none/none.go generated vendored
View File

@ -34,6 +34,19 @@ func (noCache) UncompressedDigest(anyDigest digest.Digest) digest.Digest {
func (noCache) RecordDigestUncompressedPair(anyDigest digest.Digest, uncompressed digest.Digest) {
}
// UncompressedDigestForTOC returns an uncompressed digest corresponding to anyDigest.
// Returns "" if the uncompressed digest is unknown.
func (noCache) UncompressedDigestForTOC(tocDigest digest.Digest) digest.Digest {
return ""
}
// RecordTOCUncompressedPair records that the tocDigest corresponds to uncompressed.
// WARNING: Only call this for LOCALLY VERIFIED data; don’t record a digest pair just because some remote author claims so (e.g.
// because a manifest/config pair exists); otherwise the cache could be poisoned and allow substituting unexpected blobs.
// (Eventually, the DiffIDs in image config could detect the substitution, but that may be too late, and not all image formats contain that data.)
func (noCache) RecordTOCUncompressedPair(tocDigest digest.Digest, uncompressed digest.Digest) {
}
// RecordKnownLocation records that a blob with the specified digest exists within the specified (transport, scope) scope,
// and can be reused given the opaque location data.
func (noCache) RecordKnownLocation(transport types.ImageTransport, scope types.BICTransportScope, blobDigest digest.Digest, location types.BICLocationReference) {

107
vendor/github.com/containers/image/v5/pkg/blobinfocache/sqlite/sqlite.go generated vendored
View File

@ -295,6 +295,14 @@ func ensureDBHasCurrentSchema(db *sql.DB) error {
`PRIMARY KEY (transport, scope, digest, location)
)`,
},
{
"DigestTOCUncompressedPairs",
`CREATE TABLE IF NOT EXISTS DigestTOCUncompressedPairs(` +
// index implied by PRIMARY KEY
`tocDigest TEXT PRIMARY KEY NOT NULL,` +
`uncompressedDigest TEXT NOT NULL
)`,
},
}
_, err := dbTransaction(db, func(tx *sql.Tx) (void, error) {
@ -385,6 +393,57 @@ func (sqc *cache) RecordDigestUncompressedPair(anyDigest digest.Digest, uncompre
}) // FIXME? Log error (but throttle the log volume on repeated accesses)?
}
// UncompressedDigestForTOC returns an uncompressed digest corresponding to anyDigest.
// Returns "" if the uncompressed digest is unknown.
func (sqc *cache) UncompressedDigestForTOC(tocDigest digest.Digest) digest.Digest {
res, err := transaction(sqc, func(tx *sql.Tx) (digest.Digest, error) {
uncompressedString, found, err := querySingleValue[string](tx, "SELECT uncompressedDigest FROM DigestTOCUncompressedPairs WHERE tocDigest = ?", tocDigest.String())
if err != nil {
return "", err
}
if found {
d, err := digest.Parse(uncompressedString)
if err != nil {
return "", err
}
return d, nil
}
return "", nil
})
if err != nil {
return "" // FIXME? Log err (but throttle the log volume on repeated accesses)?
}
return res
}
// RecordTOCUncompressedPair records that the tocDigest corresponds to uncompressed.
// WARNING: Only call this for LOCALLY VERIFIED data; don’t record a digest pair just because some remote author claims so (e.g.
// because a manifest/config pair exists); otherwise the cache could be poisoned and allow substituting unexpected blobs.
// (Eventually, the DiffIDs in image config could detect the substitution, but that may be too late, and not all image formats contain that data.)
func (sqc *cache) RecordTOCUncompressedPair(tocDigest digest.Digest, uncompressed digest.Digest) {
_, _ = transaction(sqc, func(tx *sql.Tx) (void, error) {
previousString, gotPrevious, err := querySingleValue[string](tx, "SELECT uncompressedDigest FROM DigestTOCUncompressedPairs WHERE tocDigest = ?", tocDigest.String())
if err != nil {
return void{}, fmt.Errorf("looking for uncompressed digest for blob with TOC %q", tocDigest)
}
if gotPrevious {
previous, err := digest.Parse(previousString)
if err != nil {
return void{}, err
}
if previous != uncompressed {
logrus.Warnf("Uncompressed digest for blob with TOC %q previously recorded as %q, now %q", tocDigest, previous, uncompressed)
}
}
if _, err := tx.Exec("INSERT OR REPLACE INTO DigestTOCUncompressedPairs(tocDigest, uncompressedDigest) VALUES (?, ?)",
tocDigest.String(), uncompressed.String()); err != nil {
return void{}, fmt.Errorf("recording uncompressed digest %q for blob with TOC %q: %w", uncompressed, tocDigest, err)
}
return void{}, nil
}) // FIXME? Log error (but throttle the log volume on repeated accesses)?
}
// RecordKnownLocation records that a blob with the specified digest exists within the specified (transport, scope) scope,
// and can be reused given the opaque location data.
func (sqc *cache) RecordKnownLocation(transport types.ImageTransport, scope types.BICTransportScope, digest digest.Digest, location types.BICLocationReference) {
@ -398,29 +457,30 @@ func (sqc *cache) RecordKnownLocation(transport types.ImageTransport, scope type
}) // FIXME? Log error (but throttle the log volume on repeated accesses)?
}
// RecordDigestCompressorName records a compressor for the blob with the specified digest,
// or Uncompressed or UnknownCompression.
// WARNING: Only call this with LOCALLY VERIFIED data; don’t record a compressor for a
// digest just because some remote author claims so (e.g. because a manifest says so);
// RecordDigestCompressorData records data for the blob with the specified digest.
// WARNING: Only call this with LOCALLY VERIFIED data:
// - don’t record a compressor for a digest just because some remote author claims so
// (e.g. because a manifest says so);
//
// otherwise the cache could be poisoned and cause us to make incorrect edits to type
// information in a manifest.
func (sqc *cache) RecordDigestCompressorName(anyDigest digest.Digest, compressorName string) {
func (sqc *cache) RecordDigestCompressorData(anyDigest digest.Digest, data blobinfocache.DigestCompressorData) {
_, _ = transaction(sqc, func(tx *sql.Tx) (void, error) {
previous, gotPrevious, err := querySingleValue[string](tx, "SELECT compressor FROM DigestCompressors WHERE digest = ?", anyDigest.String())
if err != nil {
return void{}, fmt.Errorf("looking for compressor of for %q", anyDigest)
}
if gotPrevious && previous != compressorName {
logrus.Warnf("Compressor for blob with digest %s previously recorded as %s, now %s", anyDigest, previous, compressorName)
if gotPrevious && previous != data.BaseVariantCompressor {
logrus.Warnf("Compressor for blob with digest %s previously recorded as %s, now %s", anyDigest, previous, data.BaseVariantCompressor)
}
if compressorName == blobinfocache.UnknownCompression {
if data.BaseVariantCompressor == blobinfocache.UnknownCompression {
if _, err := tx.Exec("DELETE FROM DigestCompressors WHERE digest = ?", anyDigest.String()); err != nil {
return void{}, fmt.Errorf("deleting compressor for digest %q: %w", anyDigest, err)
}
} else {
if _, err := tx.Exec("INSERT OR REPLACE INTO DigestCompressors(digest, compressor) VALUES (?, ?)",
anyDigest.String(), compressorName); err != nil {
return void{}, fmt.Errorf("recording compressor %q for %q: %w", compressorName, anyDigest, err)
anyDigest.String(), data.BaseVariantCompressor); err != nil {
return void{}, fmt.Errorf("recording compressor %q for %q: %w", data.BaseVariantCompressor, anyDigest, err)
}
}
return void{}, nil
@ -443,8 +503,10 @@ func (sqc *cache) appendReplacementCandidates(candidates []prioritize.CandidateW
compressorName = compressor
}
}
ok, compressionOp, compressionAlgo := prioritize.CandidateCompression(v2Options, digest, compressorName)
if !ok {
template := prioritize.CandidateTemplateWithCompression(v2Options, digest, blobinfocache.DigestCompressorData{
BaseVariantCompressor: compressorName,
})
if template == nil {
return candidates, nil
}
@ -463,15 +525,7 @@ func (sqc *cache) appendReplacementCandidates(candidates []prioritize.CandidateW
if err := rows.Scan(&location, &time); err != nil {
return nil, fmt.Errorf("scanning candidate: %w", err)
}
candidates = append(candidates, prioritize.CandidateWithTime{
Candidate: blobinfocache.BICReplacementCandidate2{
Digest: digest,
CompressionOperation: compressionOp,
CompressionAlgorithm: compressionAlgo,
Location: types.BICLocationReference{Opaque: location},
},
LastSeen: time,
})
candidates = append(candidates, template.CandidateWithLocation(types.BICLocationReference{Opaque: location}, time))
rowAdded = true
}
if err := rows.Err(); err != nil {
@ -479,16 +533,7 @@ func (sqc *cache) appendReplacementCandidates(candidates []prioritize.CandidateW
}
if !rowAdded && v2Options != nil {
candidates = append(candidates, prioritize.CandidateWithTime{
Candidate: blobinfocache.BICReplacementCandidate2{
Digest: digest,
CompressionOperation: compressionOp,
CompressionAlgorithm: compressionAlgo,
UnknownLocation: true,
Location: types.BICLocationReference{Opaque: ""},
},
LastSeen: time.Time{},
})
candidates = append(candidates, template.CandidateWithUnknownLocation())
}
return candidates, nil
}

6
vendor/github.com/containers/image/v5/pkg/shortnames/shortnames.go generated vendored
View File

@ -61,14 +61,12 @@ func parseUnnormalizedShortName(input string) (bool, reference.Named, error) {
// the tag or digest and stores it in the return values so that both can be
// re-added to a possible resolved alias' or USRs at a later point.
func splitUserInput(named reference.Named) (isTagged bool, isDigested bool, normalized reference.Named, tag string, digest digest.Digest) {
tagged, isT := named.(reference.NamedTagged)
if isT {
if tagged, ok := named.(reference.NamedTagged); ok {
isTagged = true
tag = tagged.Tag()
}
digested, isD := named.(reference.Digested)
if isD {
if digested, ok := named.(reference.Digested); ok {
isDigested = true
digest = digested.Digest()
}

378
vendor/github.com/containers/image/v5/storage/storage_dest.go generated vendored
View File

@ -84,18 +84,36 @@ type storageImageDestinationLockProtected struct {
currentIndex int // The index of the layer to be committed (i.e., lower indices have already been committed)
indexToAddedLayerInfo map[int]addedLayerInfo // Mapping from layer (by index) to blob to add to the image
// In general, a layer is identified either by (compressed) digest, or by TOC digest.
// Externally, a layer is identified either by (compressed) digest, or by TOC digest
// (and we assume the TOC digest also uniquely identifies the contents, i.e. there aren’t two
// different formats/ways to parse a single TOC); internally, we use uncompressed digest (“DiffID”) or a TOC digest.
// We may or may not know the relationships between these three values.
//
// When creating a layer, the c/storage layer metadata and image IDs must _only_ be based on trusted values
// we have computed ourselves. (Layer reuse can then look up against such trusted values, but it might not
// recompute those values for incomding layers — the point of the reuse is that we don’t need to consume the incoming layer.)
// Layer identification: For a layer, at least one of indexToTOCDigest and blobDiffIDs must be available before commitLayer is called.
// The presence of an indexToTOCDigest is what decides how the layer is identified, i.e. which fields must be trusted.
blobDiffIDs map[digest.Digest]digest.Digest // Mapping from layer blobsums to their corresponding DiffIDs
indexToTOCDigest map[int]digest.Digest // Mapping from layer index to a TOC Digest, IFF the layer was created/found/reused by TOC digest
// recompute those values for incoming layers — the point of the reuse is that we don’t need to consume the incoming layer.)
//
// Layer identification: For a layer, at least one of (indexToDiffID, indexToTOCDigest, blobDiffIDs) must be available
// before commitLayer is called.
// The layer is identified by the first of the three fields which exists, in that order (and the value must be trusted).
//
// WARNING: All values in indexToDiffID, indexToTOCDigest, and blobDiffIDs are _individually_ trusted, but blobDiffIDs is more subtle.
// The values in indexTo* are all consistent, because the code writing them processed them all at once, and consistently.
// But it is possible for a layer’s indexToDiffID an indexToTOCDigest to be based on a TOC, without setting blobDiffIDs
// for the compressed digest of that index, and for blobDiffIDs[compressedDigest] to be set _separately_ while processing some
// other layer entry. In particular it is possible for indexToDiffID[index] and blobDiffIDs[compressedDigestAtIndex]] to refer
// to mismatching contents.
// Users of these fields should use trustedLayerIdentityDataLocked, which centralizes the validity logic,
// instead of interpreting these fields, especially blobDiffIDs, directly.
//
// Ideally we wouldn’t have blobDiffIDs, and we would just keep records by index, but the public API does not require the caller
// to provide layer indices; and configs don’t have layer indices. blobDiffIDs needs to exist for those cases.
indexToDiffID map[int]digest.Digest // Mapping from layer index to DiffID
indexToTOCDigest map[int]digest.Digest // Mapping from layer index to a TOC Digest
blobDiffIDs map[digest.Digest]digest.Digest // Mapping from layer blobsums to their corresponding DiffIDs. CAREFUL: See the WARNING above.
// Layer data: Before commitLayer is called, either at least one of (diffOutputs, indexToAdditionalLayer, filenames)
// should be available; or indexToTOCDigest/blobDiffIDs should be enough to locate an existing c/storage layer.
// should be available; or indexToDiffID/indexToTOCDigest/blobDiffIDs should be enough to locate an existing c/storage layer.
// They are looked up in the order they are mentioned above.
diffOutputs map[int]*graphdriver.DriverWithDifferOutput // Mapping from layer index to a partially-pulled layer intermediate data
indexToAdditionalLayer map[int]storage.AdditionalLayer // Mapping from layer index to their corresponding additional layer
@ -145,9 +163,12 @@ func newImageDestination(sys *types.SystemContext, imageRef storageReference) (*
},
indexToStorageID: make(map[int]string),
lockProtected: storageImageDestinationLockProtected{
indexToAddedLayerInfo: make(map[int]addedLayerInfo),
blobDiffIDs: make(map[digest.Digest]digest.Digest),
indexToTOCDigest: make(map[int]digest.Digest),
indexToAddedLayerInfo: make(map[int]addedLayerInfo),
indexToDiffID: make(map[int]digest.Digest),
indexToTOCDigest: make(map[int]digest.Digest),
blobDiffIDs: make(map[digest.Digest]digest.Digest),
diffOutputs: make(map[int]*graphdriver.DriverWithDifferOutput),
indexToAdditionalLayer: make(map[int]storage.AdditionalLayer),
filenames: make(map[digest.Digest]string),
@ -323,20 +344,30 @@ func (s *storageImageDestination) PutBlobPartial(ctx context.Context, chunkAcces
s.lock.Lock()
if out.UncompressedDigest != "" {
s.lockProtected.indexToDiffID[options.LayerIndex] = out.UncompressedDigest
if out.TOCDigest != "" {
options.Cache.RecordTOCUncompressedPair(out.TOCDigest, out.UncompressedDigest)
}
// Don’t set indexToTOCDigest on this path:
// - Using UncompressedDigest allows image reuse with non-partially-pulled layers, so we want to set indexToDiffID.
// - If UncompressedDigest has been computed, that means the layer was read completely, and the TOC has been created from scratch.
// That TOC is quite unlikely to match any other TOC value.
// The computation of UncompressedDigest means the whole layer has been consumed; while doing that, chunked.GetDiffer is
// responsible for ensuring blobDigest has been validated.
if out.CompressedDigest != blobDigest {
return private.UploadedBlob{}, fmt.Errorf("internal error: ApplyDiffWithDiffer returned CompressedDigest %q not matching expected %q",
out.CompressedDigest, blobDigest)
}
s.lockProtected.blobDiffIDs[blobDigest] = out.UncompressedDigest
// So, record also information about blobDigest, that might benefit reuse.
// We trust ApplyDiffWithDiffer to validate or create both values correctly.
s.lockProtected.blobDiffIDs[blobDigest] = out.UncompressedDigest
options.Cache.RecordDigestUncompressedPair(out.CompressedDigest, out.UncompressedDigest)
} else {
// Don’t identify layers by TOC if UncompressedDigest is available.
// - Using UncompressedDigest allows image reuse with non-partially-pulled layers
// - If UncompressedDigest has been computed, that means the layer was read completely, and the TOC has been created from scratch.
// That TOC is quite unlikely to match with any other TOC value.
// Use diffID for layer identity if it is known.
if uncompressedDigest := options.Cache.UncompressedDigestForTOC(out.TOCDigest); uncompressedDigest != "" {
s.lockProtected.indexToDiffID[options.LayerIndex] = uncompressedDigest
}
s.lockProtected.indexToTOCDigest[options.LayerIndex] = out.TOCDigest
}
s.lockProtected.diffOutputs[options.LayerIndex] = out
@ -465,49 +496,40 @@ func (s *storageImageDestination) tryReusingBlobAsPending(blobDigest digest.Dige
if err != nil && !errors.Is(err, storage.ErrLayerUnknown) {
return false, private.ReusedBlob{}, fmt.Errorf(`looking for layers with digest %q: %w`, uncompressedDigest, err)
}
if len(layers) > 0 {
if size != -1 {
s.lockProtected.blobDiffIDs[blobDigest] = uncompressedDigest
return true, private.ReusedBlob{
Digest: blobDigest,
Size: size,
}, nil
}
if !options.CanSubstitute {
return false, private.ReusedBlob{}, fmt.Errorf("Internal error: options.CanSubstitute was expected to be true for blob with digest %s", blobDigest)
}
s.lockProtected.blobDiffIDs[uncompressedDigest] = uncompressedDigest
return true, private.ReusedBlob{
Digest: uncompressedDigest,
Size: layers[0].UncompressedSize,
}, nil
if found, reused := reusedBlobFromLayerLookup(layers, blobDigest, size, options); found {
s.lockProtected.blobDiffIDs[blobDigest] = uncompressedDigest
return true, reused, nil
}
}
}
if options.TOCDigest != "" && options.LayerIndex != nil {
// Check if we know which which UncompressedDigest the TOC digest resolves to, and we have a match for that.
// Prefer this over LayersByTOCDigest because we can identify the layer using UncompressedDigest, maximizing reuse.
uncompressedDigest := options.Cache.UncompressedDigestForTOC(options.TOCDigest)
if uncompressedDigest != "" {
layers, err = s.imageRef.transport.store.LayersByUncompressedDigest(uncompressedDigest)
if err != nil && !errors.Is(err, storage.ErrLayerUnknown) {
return false, private.ReusedBlob{}, fmt.Errorf(`looking for layers with digest %q: %w`, uncompressedDigest, err)
}
if found, reused := reusedBlobFromLayerLookup(layers, blobDigest, size, options); found {
s.lockProtected.indexToDiffID[*options.LayerIndex] = uncompressedDigest
reused.MatchedByTOCDigest = true
return true, reused, nil
}
}
// Check if we have a chunked layer in storage with the same TOC digest.
layers, err := s.imageRef.transport.store.LayersByTOCDigest(options.TOCDigest)
if err != nil && !errors.Is(err, storage.ErrLayerUnknown) {
return false, private.ReusedBlob{}, fmt.Errorf(`looking for layers with TOC digest %q: %w`, options.TOCDigest, err)
}
if len(layers) > 0 {
if size != -1 {
s.lockProtected.indexToTOCDigest[*options.LayerIndex] = options.TOCDigest
return true, private.ReusedBlob{
Digest: blobDigest,
Size: size,
MatchedByTOCDigest: true,
}, nil
} else if options.CanSubstitute && layers[0].UncompressedDigest != "" {
s.lockProtected.indexToTOCDigest[*options.LayerIndex] = options.TOCDigest
return true, private.ReusedBlob{
Digest: layers[0].UncompressedDigest,
Size: layers[0].UncompressedSize,
MatchedByTOCDigest: true,
}, nil
if found, reused := reusedBlobFromLayerLookup(layers, blobDigest, size, options); found {
if uncompressedDigest != "" {
s.lockProtected.indexToDiffID[*options.LayerIndex] = uncompressedDigest
}
s.lockProtected.indexToTOCDigest[*options.LayerIndex] = options.TOCDigest
reused.MatchedByTOCDigest = true
return true, reused, nil
}
}
@ -515,49 +537,137 @@ func (s *storageImageDestination) tryReusingBlobAsPending(blobDigest digest.Dige
return false, private.ReusedBlob{}, nil
}
// reusedBlobFromLayerLookup returns (true, ReusedBlob) if layers contain a usable match; or (false, ...) if not.
// The caller is still responsible for setting the layer identification fields, to allow the layer to be found again.
func reusedBlobFromLayerLookup(layers []storage.Layer, blobDigest digest.Digest, blobSize int64, options *private.TryReusingBlobOptions) (bool, private.ReusedBlob) {
if len(layers) > 0 {
if blobSize != -1 {
return true, private.ReusedBlob{
Digest: blobDigest,
Size: blobSize,
}
} else if options.CanSubstitute && layers[0].UncompressedDigest != "" {
return true, private.ReusedBlob{
Digest: layers[0].UncompressedDigest,
Size: layers[0].UncompressedSize,
CompressionOperation: types.Decompress,
CompressionAlgorithm: nil,
}
}
}
return false, private.ReusedBlob{}
}
// trustedLayerIdentityData is a _consistent_ set of information known about a single layer.
type trustedLayerIdentityData struct {
layerIdentifiedByTOC bool // true if we decided the layer should be identified by tocDigest, false if by diffID
diffID digest.Digest // A digest of the uncompressed full contents of the layer, or "" if unknown; must be set if !layerIdentifiedByTOC
tocDigest digest.Digest // A digest of the TOC digest, or "" if unknown; must be set if layerIdentifiedByTOC
blobDigest digest.Digest // A digest of the (possibly-compressed) layer as presented, or "" if unknown/untrusted.
}
// trustedLayerIdentityDataLocked returns a _consistent_ set of information for a layer with (layerIndex, blobDigest).
// blobDigest is the (possibly-compressed) layer digest referenced in the manifest.
// It returns (trusted, true) if the layer was found, or (_, false) if insufficient data is available.
//
// The caller must hold s.lock.
func (s *storageImageDestination) trustedLayerIdentityDataLocked(layerIndex int, blobDigest digest.Digest) (trustedLayerIdentityData, bool) {
// The decision about layerIdentifiedByTOC must be _stable_ once the data for layerIndex is set,
// even if s.lockProtected.blobDiffIDs changes later and we can subsequently find an entry that wasn’t originally available.
//
// If we previously didn't have a blobDigest match and decided to use the TOC, but _later_ we happen to find
// a blobDigest match, we might in principle want to reconsider, set layerIdentifiedByTOC to false, and use the file:
// but the layer in question, and possibly child layers, might already have been committed to storage.
// A late-arriving addition to s.lockProtected.blobDiffIDs would mean that we would want to set
// new layer IDs for potentially the whole parent chain = throw away the just-created layers and create them all again.
//
// Such a within-image layer reuse is expected to be pretty rare; instead, ignore the unexpected file match
// and proceed to the originally-planned TOC match.
res := trustedLayerIdentityData{}
diffID, layerIdentifiedByDiffID := s.lockProtected.indexToDiffID[layerIndex]
if layerIdentifiedByDiffID {
res.layerIdentifiedByTOC = false
res.diffID = diffID
}
if tocDigest, ok := s.lockProtected.indexToTOCDigest[layerIndex]; ok {
res.tocDigest = tocDigest
if !layerIdentifiedByDiffID {
res.layerIdentifiedByTOC = true
}
}
if otherDiffID, ok := s.lockProtected.blobDiffIDs[blobDigest]; ok {
if !layerIdentifiedByDiffID && !res.layerIdentifiedByTOC {
// This is the only data we have, so it is clearly self-consistent.
res.layerIdentifiedByTOC = false
res.diffID = otherDiffID
res.blobDigest = blobDigest
layerIdentifiedByDiffID = true
} else {
// We have set up the layer identity without referring to blobDigest:
// an attacker might have used a manifest with non-matching tocDigest and blobDigest.
// But, if we know a trusted diffID value from other sources, and it matches the one for blobDigest,
// we know blobDigest is fine as well.
if res.diffID != "" && otherDiffID == res.diffID {
res.blobDigest = blobDigest
}
}
}
if !layerIdentifiedByDiffID && !res.layerIdentifiedByTOC {
return trustedLayerIdentityData{}, false // We found nothing at all
}
return res, true
}
// computeID computes a recommended image ID based on information we have so far. If
// the manifest is not of a type that we recognize, we return an empty value, indicating
// that since we don't have a recommendation, a random ID should be used if one needs
// to be allocated.
func (s *storageImageDestination) computeID(m manifest.Manifest) string {
func (s *storageImageDestination) computeID(m manifest.Manifest) (string, error) {
// This is outside of the scope of HasThreadSafePutBlob, so we don’t need to hold s.lock.
layerInfos := m.LayerInfos()
// Build the diffID list. We need the decompressed sums that we've been calculating to
// fill in the DiffIDs. It's expected (but not enforced by us) that the number of
// diffIDs corresponds to the number of non-EmptyLayer entries in the history.
var diffIDs []digest.Digest
switch m := m.(type) {
switch m.(type) {
case *manifest.Schema1:
// Build a list of the diffIDs we've generated for the non-throwaway FS layers,
// in reverse of the order in which they were originally listed.
for i, compat := range m.ExtractedV1Compatibility {
if compat.ThrowAway {
// Build a list of the diffIDs we've generated for the non-throwaway FS layers
for i, li := range layerInfos {
if li.EmptyLayer {
continue
}
blobSum := m.FSLayers[i].BlobSum
diffID, ok := s.lockProtected.blobDiffIDs[blobSum]
if !ok {
// this can, in principle, legitimately happen when a layer is reused by TOC.
logrus.Infof("error looking up diffID for layer %q", blobSum.String())
return ""
trusted, ok := s.trustedLayerIdentityDataLocked(i, li.Digest)
if !ok { // We have already committed all layers if we get to this point, so the data must have been available.
return "", fmt.Errorf("internal inconsistency: layer (%d, %q) not found", i, li.Digest)
}
diffIDs = append([]digest.Digest{diffID}, diffIDs...)
if trusted.diffID == "" {
if trusted.layerIdentifiedByTOC {
logrus.Infof("v2s1 image uses a layer identified by TOC with unknown diffID; choosing a random image ID")
return "", nil
}
return "", fmt.Errorf("internal inconsistency: layer (%d, %q) is not identified by TOC and has no diffID", i, li.Digest)
}
diffIDs = append(diffIDs, trusted.diffID)
}
case *manifest.Schema2, *manifest.OCI1:
// We know the ID calculation doesn't actually use the diffIDs, so we don't need to populate
// the diffID list.
default:
return ""
return "", nil
}
// We want to use the same ID for “the same” images, but without risking unwanted sharing / malicious image corruption.
//
// Traditionally that means the same ~config digest, as computed by m.ImageID;
// but if we pull a layer by TOC, we verify the layer against neither the (compressed) blob digest in the manifest,
// but if we identify a layer by TOC, we verify the layer against neither the (compressed) blob digest in the manifest,
// nor against the config’s RootFS.DiffIDs. We don’t really want to do either, to allow partial layer pulls where we never see
// most of the data.
//
// So, if a layer is pulled by TOC (and we do validate against the TOC), the fact that we used the TOC, and the value of the TOC,
// So, if a layer is identified by TOC (and we do validate against the TOC), the fact that we used the TOC, and the value of the TOC,
// must enter into the image ID computation.
// But for images where no TOC was used, continue to use IDs computed the traditional way, to maximize image reuse on upgrades,
// and to introduce the changed behavior only when partial pulls are used.
@ -566,28 +676,31 @@ func (s *storageImageDestination) computeID(m manifest.Manifest) string {
// (skopeo copy --format v2s2 docker://…/zstd-chunked-image containers-storage:… ). So this is not happening only in the OCI case above.
ordinaryImageID, err := m.ImageID(diffIDs)
if err != nil {
return ""
return "", err
}
tocIDInput := ""
hasLayerPulledByTOC := false
for i := range m.LayerInfos() {
layerValue := "" // An empty string is not a valid digest, so this is unambiguous with the TOC case.
tocDigest, ok := s.lockProtected.indexToTOCDigest[i] // "" if not a TOC
if ok {
for i, li := range layerInfos {
trusted, ok := s.trustedLayerIdentityDataLocked(i, li.Digest)
if !ok { // We have already committed all layers if we get to this point, so the data must have been available.
return "", fmt.Errorf("internal inconsistency: layer (%d, %q) not found", i, li.Digest)
}
layerValue := "" // An empty string is not a valid digest, so this is unambiguous with the TOC case.
if trusted.layerIdentifiedByTOC {
hasLayerPulledByTOC = true
layerValue = tocDigest.String()
layerValue = trusted.tocDigest.String()
}
tocIDInput += layerValue + "|" // "|" can not be present in a TOC digest, so this is an unambiguous separator.
}
if !hasLayerPulledByTOC {
return ordinaryImageID
return ordinaryImageID, nil
}
// ordinaryImageID is a digest of a config, which is a JSON value.
// To avoid the risk of collisions, start the input with @ so that the input is not a valid JSON.
tocImageID := digest.FromString("@With TOC:" + tocIDInput).Encoded()
logrus.Debugf("Ordinary storage image ID %s; a layer was looked up by TOC, so using image ID %s", ordinaryImageID, tocImageID)
return tocImageID
return tocImageID, nil
}
// getConfigBlob exists only to let us retrieve the configuration blob so that the manifest package can dig
@ -671,14 +784,14 @@ func (s *storageImageDestination) singleLayerIDComponent(layerIndex int, blobDig
s.lock.Lock()
defer s.lock.Unlock()
if d, found := s.lockProtected.indexToTOCDigest[layerIndex]; found {
return "@TOC=" + d.Encoded(), false // "@" is not a valid start of a digest.Digest, so this is unambiguous.
trusted, ok := s.trustedLayerIdentityDataLocked(layerIndex, blobDigest)
if !ok {
return "", false
}
if d, found := s.lockProtected.blobDiffIDs[blobDigest]; found {
return d.Encoded(), true // This looks like chain IDs, and it uses the traditional value.
if trusted.layerIdentifiedByTOC {
return "@TOC=" + trusted.tocDigest.Encoded(), false // "@" is not a valid start of a digest.Digest, so this is unambiguous.
}
return "", false
return trusted.diffID.Encoded(), true // This looks like chain IDs, and it uses the traditional value.
}
// commitLayer commits the specified layer with the given index to the storage.
@ -778,6 +891,16 @@ func (s *storageImageDestination) createNewLayer(index int, layerDigest digest.D
diffOutput, ok := s.lockProtected.diffOutputs[index]
s.lock.Unlock()
if ok {
// If we know a trusted DiffID value (e.g. from a BlobInfoCache), set it in diffOutput.
// That way it will be persisted in storage even if the cache is deleted; also
// we can use the value below to avoid the untrustedUncompressedDigest logic (and notably
// the costly commit delay until a manifest is available).
s.lock.Lock()
if d, ok := s.lockProtected.indexToDiffID[index]; ok {
diffOutput.UncompressedDigest = d
}
s.lock.Unlock()
var untrustedUncompressedDigest digest.Digest
if diffOutput.UncompressedDigest == "" {
d, err := s.untrustedLayerDiffID(index)
@ -832,47 +955,43 @@ func (s *storageImageDestination) createNewLayer(index int, layerDigest digest.D
// Check if we previously cached a file with that blob's contents. If we didn't,
// then we need to read the desired contents from a layer.
var trustedUncompressedDigest, trustedOriginalDigest digest.Digest // For storage.LayerOptions
var filename string
var gotFilename bool
s.lock.Lock()
tocDigest := s.lockProtected.indexToTOCDigest[index] // "" if not set
optionalDiffID := s.lockProtected.blobDiffIDs[layerDigest] // "" if not set
filename, gotFilename := s.lockProtected.filenames[layerDigest]
trusted, ok := s.trustedLayerIdentityDataLocked(index, layerDigest)
if ok && trusted.blobDigest != "" {
filename, gotFilename = s.lockProtected.filenames[trusted.blobDigest]
}
s.lock.Unlock()
if gotFilename && tocDigest == "" {
// If tocDigest != "", if we now happen to find a layerDigest match, the newLayerID has already been computed as TOC-based,
// and we don't know the relationship of the layerDigest and TOC digest.
// We could recompute newLayerID to be DiffID-based and use the file, but such a within-image layer
// reuse is expected to be pretty rare; instead, ignore the unexpected file match and proceed to the
// originally-planned TOC match.
// Because tocDigest == "", optionaldiffID must have been set; and even if it weren’t, PutLayer will recompute the digest from the stream.
trustedUncompressedDigest = optionalDiffID
trustedOriginalDigest = layerDigest // The code setting .filenames[layerDigest] is responsible for the contents matching.
if !ok { // We have already determined newLayerID, so the data must have been available.
return nil, fmt.Errorf("internal inconsistency: layer (%d, %q) not found", index, layerDigest)
}
var trustedOriginalDigest digest.Digest // For storage.LayerOptions
if gotFilename {
// The code setting .filenames[trusted.blobDigest] is responsible for ensuring that the file contents match trusted.blobDigest.
trustedOriginalDigest = trusted.blobDigest
} else {
// Try to find the layer with contents matching the data we use.
var layer *storage.Layer // = nil
if tocDigest != "" {
layers, err2 := s.imageRef.transport.store.LayersByTOCDigest(tocDigest)
if err2 == nil && len(layers) > 0 {
if trusted.diffID != "" {
if layers, err2 := s.imageRef.transport.store.LayersByUncompressedDigest(trusted.diffID); err2 == nil && len(layers) > 0 {
layer = &layers[0]
} else {
return nil, fmt.Errorf("locating layer for TOC digest %q: %w", tocDigest, err2)
}
} else {
// Because tocDigest == "", optionaldiffID must have been set
layers, err2 := s.imageRef.transport.store.LayersByUncompressedDigest(optionalDiffID)
if err2 == nil && len(layers) > 0 {
layer = &layers[0]
} else {
layers, err2 = s.imageRef.transport.store.LayersByCompressedDigest(layerDigest)
if err2 == nil && len(layers) > 0 {
layer = &layers[0]
}
}
if layer == nil {
return nil, fmt.Errorf("locating layer for blob %q: %w", layerDigest, err2)
}
}
if layer == nil && trusted.tocDigest != "" {
if layers, err2 := s.imageRef.transport.store.LayersByTOCDigest(trusted.tocDigest); err2 == nil && len(layers) > 0 {
layer = &layers[0]
}
}
if layer == nil && trusted.blobDigest != "" {
if layers, err2 := s.imageRef.transport.store.LayersByCompressedDigest(trusted.blobDigest); err2 == nil && len(layers) > 0 {
layer = &layers[0]
}
}
if layer == nil {
return nil, fmt.Errorf("layer for blob %q/%q/%q not found", trusted.blobDigest, trusted.tocDigest, trusted.diffID)
}
// Read the layer's contents.
noCompression := archive.Uncompressed
diffOptions := &storage.DiffOptions{
@ -880,7 +999,7 @@ func (s *storageImageDestination) createNewLayer(index int, layerDigest digest.D
}
diff, err2 := s.imageRef.transport.store.Diff("", layer.ID, diffOptions)
if err2 != nil {
return nil, fmt.Errorf("reading layer %q for blob %q: %w", layer.ID, layerDigest, err2)
return nil, fmt.Errorf("reading layer %q for blob %q/%q/%q: %w", layer.ID, trusted.blobDigest, trusted.tocDigest, trusted.diffID, err2)
}
// Copy the layer diff to a file. Diff() takes a lock that it holds
// until the ReadCloser that it returns is closed, and PutLayer() wants
@ -902,20 +1021,19 @@ func (s *storageImageDestination) createNewLayer(index int, layerDigest digest.D
return nil, fmt.Errorf("storing blob to file %q: %w", filename, err)
}
if optionalDiffID == "" && layer.UncompressedDigest != "" {
optionalDiffID = layer.UncompressedDigest
if trusted.diffID == "" && layer.UncompressedDigest != "" {
trusted.diffID = layer.UncompressedDigest // This data might have been unavailable in tryReusingBlobAsPending, and is only known now.
}
// The stream we have is uncompressed, this matches contents of the stream.
// If tocDigest != "", trustedUncompressedDigest might still be ""; in that case PutLayer will compute the value from the stream.
trustedUncompressedDigest = optionalDiffID
// FIXME? trustedOriginalDigest could be set to layerDigest IF tocDigest == "" (otherwise layerDigest is untrusted).
// The stream we have is uncompressed, and it matches trusted.diffID (if known).
//
// FIXME? trustedOriginalDigest could be set to trusted.blobDigest if known, to allow more layer reuse.
// But for c/storage to reasonably use it (as a CompressedDigest value), we should also ensure the CompressedSize of the created
// layer is correct, and the API does not currently make it possible (.CompressedSize is set from the input stream).
//
// We can legitimately set storage.LayerOptions.OriginalDigest to "",
// but that would just result in PutLayer computing the digest of the input stream == optionalDiffID.
// but that would just result in PutLayer computing the digest of the input stream == trusted.diffID.
// So, instead, set .OriginalDigest to the value we know already, to avoid that digest computation.
trustedOriginalDigest = optionalDiffID
trustedOriginalDigest = trusted.diffID
// Allow using the already-collected layer contents without extracting the layer again.
//
@ -923,11 +1041,11 @@ func (s *storageImageDestination) createNewLayer(index int, layerDigest digest.D
// We don’t have the original compressed data here to trivially set filenames[layerDigest].
// In particular we can’t achieve the correct Layer.CompressedSize value with the current c/storage API.
// Within-image layer reuse is probably very rare, for now we prefer to avoid that complexity.
if trustedUncompressedDigest != "" {
if trusted.diffID != "" {
s.lock.Lock()
s.lockProtected.blobDiffIDs[trustedUncompressedDigest] = trustedUncompressedDigest
s.lockProtected.filenames[trustedUncompressedDigest] = filename
s.lockProtected.fileSizes[trustedUncompressedDigest] = fileSize
s.lockProtected.blobDiffIDs[trusted.diffID] = trusted.diffID
s.lockProtected.filenames[trusted.diffID] = filename
s.lockProtected.fileSizes[trusted.diffID] = fileSize
s.lock.Unlock()
}
}
@ -940,11 +1058,12 @@ func (s *storageImageDestination) createNewLayer(index int, layerDigest digest.D
// Build the new layer using the diff, regardless of where it came from.
// TODO: This can take quite some time, and should ideally be cancellable using ctx.Done().
layer, _, err := s.imageRef.transport.store.PutLayer(newLayerID, parentLayer, nil, "", false, &storage.LayerOptions{
OriginalDigest: trustedOriginalDigest,
UncompressedDigest: trustedUncompressedDigest,
OriginalDigest: trustedOriginalDigest,
// This might be "" if trusted.layerIdentifiedByTOC; in that case PutLayer will compute the value from the stream.
UncompressedDigest: trusted.diffID,
}, file)
if err != nil && !errors.Is(err, storage.ErrDuplicateID) {
return nil, fmt.Errorf("adding layer with blob %q: %w", layerDigest, err)
return nil, fmt.Errorf("adding layer with blob %q/%q/%q: %w", trusted.blobDigest, trusted.tocDigest, trusted.diffID, err)
}
return layer, nil
}
@ -1155,7 +1274,10 @@ func (s *storageImageDestination) Commit(ctx context.Context, unparsedToplevel t
// Create the image record, pointing to the most-recently added layer.
intendedID := s.imageRef.id
if intendedID == "" {
intendedID = s.computeID(man)
intendedID, err = s.computeID(man)
if err != nil {
return err
}
}
oldNames := []string{}
img, err := s.imageRef.transport.store.CreateImage(intendedID, nil, lastLayer, "", options)

4
vendor/github.com/containers/image/v5/version/version.go generated vendored
View File

@ -6,12 +6,12 @@ const (
// VersionMajor is for an API incompatible changes
VersionMajor = 5
// VersionMinor is for functionality in a backwards-compatible manner
VersionMinor = 32
VersionMinor = 33
// VersionPatch is for backwards-compatible bug fixes
VersionPatch = 0
// VersionDev indicates development branch. Releases will be empty string.
VersionDev = ""
VersionDev = "-dev"
)
// Version is the specification version that the package types support.

4
vendor/golang.org/x/oauth2/LICENSE generated vendored
View File

@ -1,4 +1,4 @@
Copyright (c) 2009 The Go Authors. All rights reserved.
Copyright 2009 The Go Authors.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
@ -10,7 +10,7 @@ notice, this list of conditions and the following disclaimer.
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
* Neither the name of Google LLC nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.

6
vendor/modules.txt vendored
View File

@ -170,7 +170,7 @@ github.com/containers/buildah/pkg/sshagent
github.com/containers/buildah/pkg/util
github.com/containers/buildah/pkg/volumes
github.com/containers/buildah/util
# github.com/containers/common v0.60.1-0.20240806133927-05b2e1fd7cd0
# github.com/containers/common v0.60.1-0.20240808214705-d93f74f43223
## explicit; go 1.21.0
github.com/containers/common/internal
github.com/containers/common/internal/attributedstring
@ -243,7 +243,7 @@ github.com/containers/conmon/runner/config
# github.com/containers/gvisor-tap-vsock v0.7.4
## explicit; go 1.21
github.com/containers/gvisor-tap-vsock/pkg/types
# github.com/containers/image/v5 v5.32.0
# github.com/containers/image/v5 v5.32.1-0.20240806084436-e3e9287ca8e6
## explicit; go 1.21.0
github.com/containers/image/v5/copy
github.com/containers/image/v5/directory
@ -1202,7 +1202,7 @@ golang.org/x/net/internal/socks
golang.org/x/net/internal/timeseries
golang.org/x/net/proxy
golang.org/x/net/trace
# golang.org/x/oauth2 v0.21.0
# golang.org/x/oauth2 v0.22.0
## explicit; go 1.18
golang.org/x/oauth2
golang.org/x/oauth2/internal