diff --git a/go.mod b/go.mod
index 600c9dbf2e..0461e55d49 100644
--- a/go.mod
+++ b/go.mod
@@ -20,7 +20,7 @@ require (
github.com/containers/libhvee v0.4.1-0.20231012183749-e51be96b4854
github.com/containers/ocicrypt v1.1.8
github.com/containers/psgo v1.8.0
- github.com/containers/storage v1.50.3-0.20231005200628-e21971a94abb
+ github.com/containers/storage v1.50.3-0.20231019074621-79aa304201ff
github.com/coreos/go-systemd/v22 v22.5.0
github.com/coreos/stream-metadata-go v0.4.3
github.com/crc-org/vfkit v0.1.2-0.20230829083117-09e62065eb6e
@@ -139,7 +139,7 @@ require (
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/jinzhu/copier v0.4.0 // indirect
github.com/josharian/intern v1.0.0 // indirect
- github.com/klauspost/compress v1.17.0 // indirect
+ github.com/klauspost/compress v1.17.1 // indirect
github.com/klauspost/cpuid/v2 v2.2.4 // indirect
github.com/klauspost/pgzip v1.2.6 // indirect
github.com/kr/fs v0.1.0 // indirect
diff --git a/go.sum b/go.sum
index aa7d875b86..05ffee7f18 100644
--- a/go.sum
+++ b/go.sum
@@ -273,8 +273,8 @@ github.com/containers/ocicrypt v1.1.8/go.mod h1:jM362hyBtbwLMWzXQZTlkjKGAQf/BN/L
github.com/containers/psgo v1.8.0 h1:2loGekmGAxM9ir5OsXWEfGwFxorMPYnc6gEDsGFQvhY=
github.com/containers/psgo v1.8.0/go.mod h1:T8ZxnX3Ur4RvnhxFJ7t8xJ1F48RhiZB4rSrOaR/qGHc=
github.com/containers/storage v1.43.0/go.mod h1:uZ147thiIFGdVTjMmIw19knttQnUCl3y9zjreHrg11s=
-github.com/containers/storage v1.50.3-0.20231005200628-e21971a94abb h1:fYLIShJAebUb9l2643IoD9jrj1WEyhux/4/V+2D2CXM=
-github.com/containers/storage v1.50.3-0.20231005200628-e21971a94abb/go.mod h1:RepNBJN07JUzkeLwmAeRwUstBqisvf8UoW/E6TR1Pak=
+github.com/containers/storage v1.50.3-0.20231019074621-79aa304201ff h1:/z85oN77tAw5UUEzymFH73nHIgNtGeL9YednTut9E8M=
+github.com/containers/storage v1.50.3-0.20231019074621-79aa304201ff/go.mod h1:UMgAdUgejmMKJQqeRE+0X/3GP9O6rXRm+E4uS7QrFgk=
github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
github.com/coreos/go-iptables v0.4.5/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU=
@@ -680,8 +680,8 @@ github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdY
github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
github.com/klauspost/compress v1.15.7/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
-github.com/klauspost/compress v1.17.0 h1:Rnbp4K9EjcDuVuHtd0dgA4qNuv9yKDYKK1ulpJwgrqM=
-github.com/klauspost/compress v1.17.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
+github.com/klauspost/compress v1.17.1 h1:NE3C767s2ak2bweCZo3+rdP4U/HoyVXLv/X9f2gPS5g=
+github.com/klauspost/compress v1.17.1/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk=
github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY=
diff --git a/vendor/github.com/containers/storage/drivers/driver.go b/vendor/github.com/containers/storage/drivers/driver.go
index 1fb04dc3ed..ab32d652e7 100644
--- a/vendor/github.com/containers/storage/drivers/driver.go
+++ b/vendor/github.com/containers/storage/drivers/driver.go
@@ -189,6 +189,9 @@ type DriverWithDifferOutput struct {
BigData map[string][]byte
TarSplit []byte
TOCDigest digest.Digest
+ // Artifacts is a collection of additional artifacts
+ // generated by the differ that the storage driver can use.
+ Artifacts map[string]interface{}
}
type DifferOutputFormat int
diff --git a/vendor/github.com/containers/storage/drivers/overlay/composefs_notsupported.go b/vendor/github.com/containers/storage/drivers/overlay/composefs_notsupported.go
index 5cdbcff6c4..347e4d35c4 100644
--- a/vendor/github.com/containers/storage/drivers/overlay/composefs_notsupported.go
+++ b/vendor/github.com/containers/storage/drivers/overlay/composefs_notsupported.go
@@ -11,7 +11,7 @@ func composeFsSupported() bool {
return false
}
-func generateComposeFsBlob(toc []byte, composefsDir string) error {
+func generateComposeFsBlob(verityDigests map[string]string, toc interface{}, composefsDir string) error {
return fmt.Errorf("composefs is not supported")
}
@@ -19,6 +19,6 @@ func mountComposefsBlob(dataDir, mountPoint string) error {
return fmt.Errorf("composefs is not supported")
}
-func enableVerityRecursive(path string) error {
- return fmt.Errorf("composefs is not supported")
+func enableVerityRecursive(path string) (map[string]string, error) {
+ return nil, fmt.Errorf("composefs is not supported")
}
diff --git a/vendor/github.com/containers/storage/drivers/overlay/composefs_supported.go b/vendor/github.com/containers/storage/drivers/overlay/composefs_supported.go
index 2f45c0f685..26dd36866c 100644
--- a/vendor/github.com/containers/storage/drivers/overlay/composefs_supported.go
+++ b/vendor/github.com/containers/storage/drivers/overlay/composefs_supported.go
@@ -4,7 +4,6 @@
package overlay
import (
- "bytes"
"encoding/binary"
"errors"
"fmt"
@@ -16,6 +15,7 @@ import (
"syscall"
"unsafe"
+ "github.com/containers/storage/pkg/chunked/dump"
"github.com/containers/storage/pkg/loopback"
"github.com/sirupsen/logrus"
"golang.org/x/sys/unix"
@@ -29,7 +29,7 @@ var (
func getComposeFsHelper() (string, error) {
composeFsHelperOnce.Do(func() {
- composeFsHelperPath, composeFsHelperErr = exec.LookPath("composefs-from-json")
+ composeFsHelperPath, composeFsHelperErr = exec.LookPath("mkcomposefs")
})
return composeFsHelperPath, composeFsHelperErr
}
@@ -53,7 +53,23 @@ func enableVerity(description string, fd int) error {
return nil
}
-func enableVerityRecursive(path string) error {
+type verityDigest struct {
+ Fsv unix.FsverityDigest
+ Buf [64]byte
+}
+
+func measureVerity(description string, fd int) (string, error) {
+ var digest verityDigest
+ digest.Fsv.Size = 64
+ _, _, e1 := syscall.Syscall(unix.SYS_IOCTL, uintptr(fd), uintptr(unix.FS_IOC_MEASURE_VERITY), uintptr(unsafe.Pointer(&digest)))
+ if e1 != 0 {
+ return "", fmt.Errorf("failed to measure verity for %q: %w", description, e1)
+ }
+ return fmt.Sprintf("%x", digest.Buf[:digest.Fsv.Size]), nil
+}
+
+func enableVerityRecursive(root string) (map[string]string, error) {
+ digests := make(map[string]string)
walkFn := func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
@@ -71,24 +87,42 @@ func enableVerityRecursive(path string) error {
if err := enableVerity(path, int(f.Fd())); err != nil {
return err
}
+
+ verity, err := measureVerity(path, int(f.Fd()))
+ if err != nil {
+ return err
+ }
+
+ relPath, err := filepath.Rel(root, path)
+ if err != nil {
+ return err
+ }
+
+ digests[relPath] = verity
return nil
}
- return filepath.WalkDir(path, walkFn)
+ err := filepath.WalkDir(root, walkFn)
+ return digests, err
}
func getComposefsBlob(dataDir string) string {
return filepath.Join(dataDir, "composefs.blob")
}
-func generateComposeFsBlob(toc []byte, composefsDir string) error {
+func generateComposeFsBlob(verityDigests map[string]string, toc interface{}, composefsDir string) error {
if err := os.MkdirAll(composefsDir, 0o700); err != nil {
return err
}
+ dumpReader, err := dump.GenerateDump(toc, verityDigests)
+ if err != nil {
+ return err
+ }
+
destFile := getComposefsBlob(composefsDir)
writerJson, err := getComposeFsHelper()
if err != nil {
- return fmt.Errorf("failed to find composefs-from-json: %w", err)
+ return fmt.Errorf("failed to find mkcomposefs: %w", err)
}
fd, err := unix.Openat(unix.AT_FDCWD, destFile, unix.O_WRONLY|unix.O_CREAT|unix.O_TRUNC|unix.O_EXCL|unix.O_CLOEXEC, 0o644)
@@ -109,10 +143,10 @@ func generateComposeFsBlob(toc []byte, composefsDir string) error {
// a scope to close outFd before setting fsverity on the read-only fd.
defer outFd.Close()
- cmd := exec.Command(writerJson, "--format=erofs", "--out=/proc/self/fd/3", "/proc/self/fd/0")
+ cmd := exec.Command(writerJson, "--from-file", "-", "/proc/self/fd/3")
cmd.ExtraFiles = []*os.File{outFd}
cmd.Stderr = os.Stderr
- cmd.Stdin = bytes.NewReader(toc)
+ cmd.Stdin = dumpReader
if err := cmd.Run(); err != nil {
return fmt.Errorf("failed to convert json to erofs: %w", err)
}
diff --git a/vendor/github.com/containers/storage/drivers/overlay/overlay.go b/vendor/github.com/containers/storage/drivers/overlay/overlay.go
index 525655331d..04ecf871fd 100644
--- a/vendor/github.com/containers/storage/drivers/overlay/overlay.go
+++ b/vendor/github.com/containers/storage/drivers/overlay/overlay.go
@@ -82,7 +82,7 @@ const (
lowerFile = "lower"
maxDepth = 500
- zstdChunkedManifest = "zstd-chunked-manifest"
+ tocArtifact = "toc"
// idLength represents the number of random characters
// which can be used to create the unique link identifier
@@ -1003,8 +1003,10 @@ func (d *Driver) create(id, parent string, opts *graphdriver.CreateOpts, disable
}
}
if parent != "" {
- parentBase, parentImageStore, _ := d.dir2(parent)
- if parentImageStore != "" {
+ parentBase, parentImageStore, inAdditionalStore := d.dir2(parent)
+ // If parentBase path is additional image store, select the image contained in parentBase.
+ // See https://github.com/containers/podman/issues/19748
+ if parentImageStore != "" && !inAdditionalStore {
parentBase = parentImageStore
}
st, err := system.Stat(filepath.Join(parentBase, "diff"))
@@ -1079,12 +1081,13 @@ func (d *Driver) create(id, parent string, opts *graphdriver.CreateOpts, disable
}
if parent != "" {
- parentDir, parentImageStore, _ := d.dir2(parent)
- base := parentDir
- if parentImageStore != "" {
- base = parentImageStore
+ parentBase, parentImageStore, inAdditionalStore := d.dir2(parent)
+ // If parentBase path is additional image store, select the image contained in parentBase.
+ // See https://github.com/containers/podman/issues/19748
+ if parentImageStore != "" && !inAdditionalStore {
+ parentBase = parentImageStore
}
- st, err := system.Stat(filepath.Join(base, "diff"))
+ st, err := system.Stat(filepath.Join(parentBase, "diff"))
if err != nil {
return err
}
@@ -1526,15 +1529,8 @@ func (d *Driver) get(id string, disableShifting bool, options graphdriver.MountO
defer cleanupFunc()
}
- composefsLayers := filepath.Join(workDirBase, "composefs-layers")
- if err := os.MkdirAll(composefsLayers, 0o700); err != nil {
- return "", err
- }
-
skipIDMappingLayers := make(map[string]string)
- composeFsLayers := []string{}
-
composefsMounts := []string{}
defer func() {
for _, m := range composefsMounts {
@@ -1542,6 +1538,8 @@ func (d *Driver) get(id string, disableShifting bool, options graphdriver.MountO
}
}()
+ composeFsLayers := []string{}
+ composeFsLayersDir := filepath.Join(workDirBase, "composefs-layers")
maybeAddComposefsMount := func(lowerID string, i int, readWrite bool) (string, error) {
composefsBlob := d.getComposefsData(lowerID)
_, err = os.Stat(composefsBlob)
@@ -1557,7 +1555,7 @@ func (d *Driver) get(id string, disableShifting bool, options graphdriver.MountO
return "", fmt.Errorf("cannot mount a composefs layer as writeable")
}
- dest := filepath.Join(composefsLayers, fmt.Sprintf("%d", i))
+ dest := filepath.Join(composeFsLayersDir, fmt.Sprintf("%d", i))
if err := os.MkdirAll(dest, 0o700); err != nil {
return "", err
}
@@ -2110,11 +2108,12 @@ func (d *Driver) ApplyDiffFromStagingDirectory(id, parent, stagingDirectory stri
if d.useComposeFs() {
// FIXME: move this logic into the differ so we don't have to open
// the file twice.
- if err := enableVerityRecursive(stagingDirectory); err != nil && !errors.Is(err, unix.ENOTSUP) && !errors.Is(err, unix.ENOTTY) {
+ verityDigests, err := enableVerityRecursive(stagingDirectory)
+ if err != nil && !errors.Is(err, unix.ENOTSUP) && !errors.Is(err, unix.ENOTTY) {
logrus.Warningf("%s", err)
}
- toc := diffOutput.BigData[zstdChunkedManifest]
- if err := generateComposeFsBlob(toc, d.getComposefsData(id)); err != nil {
+ toc := diffOutput.Artifacts[tocArtifact]
+ if err := generateComposeFsBlob(verityDigests, toc, d.getComposefsData(id)); err != nil {
return err
}
}
diff --git a/vendor/github.com/containers/storage/pkg/chunked/cache_linux.go b/vendor/github.com/containers/storage/pkg/chunked/cache_linux.go
index 56c30e2675..5d4befc234 100644
--- a/vendor/github.com/containers/storage/pkg/chunked/cache_linux.go
+++ b/vendor/github.com/containers/storage/pkg/chunked/cache_linux.go
@@ -26,6 +26,8 @@ import (
const (
cacheKey = "chunked-manifest-cache"
cacheVersion = 1
+
+ digestSha256Empty = "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
)
type metadata struct {
@@ -650,6 +652,9 @@ func unmarshalToc(manifest []byte) (*internal.TOC, error) {
iter.Skip()
}
}
+ if m.Type == TypeReg && m.Size == 0 && m.Digest == "" {
+ m.Digest = digestSha256Empty
+ }
toc.Entries = append(toc.Entries, m)
}
break
diff --git a/vendor/github.com/containers/storage/pkg/chunked/dump/dump.go b/vendor/github.com/containers/storage/pkg/chunked/dump/dump.go
new file mode 100644
index 0000000000..a08928034a
--- /dev/null
+++ b/vendor/github.com/containers/storage/pkg/chunked/dump/dump.go
@@ -0,0 +1,230 @@
+package dump
+
+import (
+ "bufio"
+ "fmt"
+ "io"
+ "strings"
+ "time"
+ "unicode"
+
+ "github.com/containers/storage/pkg/chunked/internal"
+ "golang.org/x/sys/unix"
+)
+
+const (
+ ESCAPE_STANDARD = 0
+ NOESCAPE_SPACE = 1 << iota
+ ESCAPE_EQUAL
+ ESCAPE_LONE_DASH
+)
+
+func escaped(val string, escape int) string {
+ noescapeSpace := escape&NOESCAPE_SPACE != 0
+ escapeEqual := escape&ESCAPE_EQUAL != 0
+ escapeLoneDash := escape&ESCAPE_LONE_DASH != 0
+
+ length := len(val)
+
+ if escapeLoneDash && val == "-" {
+ return fmt.Sprintf("\\x%.2x", val[0])
+ }
+
+ var result string
+ for i := 0; i < length; i++ {
+ c := val[i]
+ hexEscape := false
+ var special string
+
+ switch c {
+ case '\\':
+ special = "\\\\"
+ case '\n':
+ special = "\\n"
+ case '\r':
+ special = "\\r"
+ case '\t':
+ special = "\\t"
+ case '=':
+ hexEscape = escapeEqual
+ default:
+ if noescapeSpace {
+ hexEscape = !unicode.IsPrint(rune(c))
+ } else {
+ hexEscape = !unicode.IsGraphic(rune(c))
+ }
+ }
+
+ if special != "" {
+ result += special
+ } else if hexEscape {
+ result += fmt.Sprintf("\\x%.2x", c)
+ } else {
+ result += string(c)
+ }
+ }
+ return result
+}
+
+func escapedOptional(val string, escape int) string {
+ if val == "" {
+ return "-"
+ }
+ return escaped(val, escape)
+}
+
+func getStMode(mode uint32, typ string) (uint32, error) {
+ switch typ {
+ case internal.TypeReg, internal.TypeLink:
+ mode |= unix.S_IFREG
+ case internal.TypeChar:
+ mode |= unix.S_IFCHR
+ case internal.TypeBlock:
+ mode |= unix.S_IFBLK
+ case internal.TypeDir:
+ mode |= unix.S_IFDIR
+ case internal.TypeFifo:
+ mode |= unix.S_IFIFO
+ case internal.TypeSymlink:
+ mode |= unix.S_IFLNK
+ default:
+ return 0, fmt.Errorf("unknown type %s", typ)
+ }
+ return mode, nil
+}
+
+func dumpNode(out io.Writer, links map[string]int, verityDigests map[string]string, entry *internal.FileMetadata) error {
+ path := entry.Name
+ if path == "" {
+ path = "/"
+ } else if path[0] != '/' {
+ path = "/" + path
+ }
+
+ if _, err := fmt.Fprint(out, escaped(path, ESCAPE_STANDARD)); err != nil {
+ return err
+ }
+
+ nlinks := links[entry.Name] + links[entry.Linkname] + 1
+ link := ""
+ if entry.Type == internal.TypeLink {
+ link = "@"
+ }
+
+ rdev := unix.Mkdev(uint32(entry.Devmajor), uint32(entry.Devminor))
+
+ entryTime := entry.ModTime
+ if entryTime == nil {
+ t := time.Unix(0, 0)
+ entryTime = &t
+ }
+
+ mode, err := getStMode(uint32(entry.Mode), entry.Type)
+ if err != nil {
+ return err
+ }
+
+ if _, err := fmt.Fprintf(out, " %d %s%o %d %d %d %d %d.%d ", entry.Size,
+ link, mode,
+ nlinks, entry.UID, entry.GID, rdev,
+ entryTime.Unix(), entryTime.Nanosecond()); err != nil {
+ return err
+ }
+
+ var payload string
+ if entry.Linkname != "" {
+ payload = entry.Linkname
+ if entry.Type == internal.TypeLink && payload[0] != '/' {
+ payload = "/" + payload
+ }
+ } else {
+ if len(entry.Digest) > 10 {
+ d := strings.Replace(entry.Digest, "sha256:", "", 1)
+ payload = d[:2] + "/" + d[2:]
+ }
+ }
+
+ if _, err := fmt.Fprintf(out, escapedOptional(payload, ESCAPE_LONE_DASH)); err != nil {
+ return err
+ }
+
+ /* inline content. */
+ if _, err := fmt.Fprint(out, " -"); err != nil {
+ return err
+ }
+
+ /* store the digest. */
+ if _, err := fmt.Fprint(out, " "); err != nil {
+ return err
+ }
+ digest := verityDigests[payload]
+ if _, err := fmt.Fprintf(out, escapedOptional(digest, ESCAPE_LONE_DASH)); err != nil {
+ return err
+ }
+
+ for k, v := range entry.Xattrs {
+ name := escaped(k, ESCAPE_EQUAL)
+ value := escaped(v, ESCAPE_EQUAL)
+
+ if _, err := fmt.Fprintf(out, " %s=%s", name, value); err != nil {
+ return err
+ }
+ }
+ if _, err := fmt.Fprint(out, "\n"); err != nil {
+ return err
+ }
+ return nil
+}
+
+// GenerateDump generates a dump of the TOC in the same format as `composefs-info dump`
+func GenerateDump(tocI interface{}, verityDigests map[string]string) (io.Reader, error) {
+ toc, ok := tocI.(*internal.TOC)
+ if !ok {
+ return nil, fmt.Errorf("invalid TOC type")
+ }
+ pipeR, pipeW := io.Pipe()
+ go func() {
+ closed := false
+ w := bufio.NewWriter(pipeW)
+ defer func() {
+ if !closed {
+ w.Flush()
+ pipeW.Close()
+ }
+ }()
+
+ links := make(map[string]int)
+ for _, e := range toc.Entries {
+ if e.Linkname == "" {
+ continue
+ }
+ links[e.Linkname] = links[e.Linkname] + 1
+ }
+
+ if len(toc.Entries) == 0 || (toc.Entries[0].Name != "" && toc.Entries[0].Name != "/") {
+ root := &internal.FileMetadata{
+ Name: "/",
+ Type: internal.TypeDir,
+ Mode: 0o755,
+ }
+
+ if err := dumpNode(w, links, verityDigests, root); err != nil {
+ pipeW.CloseWithError(err)
+ closed = true
+ return
+ }
+ }
+
+ for _, e := range toc.Entries {
+ if e.Type == internal.TypeChunk {
+ continue
+ }
+ if err := dumpNode(w, links, verityDigests, &e); err != nil {
+ pipeW.CloseWithError(err)
+ closed = true
+ return
+ }
+ }
+ }()
+ return pipeR, nil
+}
diff --git a/vendor/github.com/containers/storage/pkg/chunked/storage_linux.go b/vendor/github.com/containers/storage/pkg/chunked/storage_linux.go
index 088c92782c..8493a2c19a 100644
--- a/vendor/github.com/containers/storage/pkg/chunked/storage_linux.go
+++ b/vendor/github.com/containers/storage/pkg/chunked/storage_linux.go
@@ -45,6 +45,7 @@ const (
bigDataKey = "zstd-chunked-manifest"
chunkedData = "zstd-chunked-data"
chunkedLayerDataKey = "zstd-chunked-layer-data"
+ tocKey = "toc"
fileTypeZstdChunked = iota
fileTypeEstargz
@@ -1470,10 +1471,7 @@ func makeEntriesFlat(mergedEntries []internal.FileMetadata) ([]internal.FileMeta
continue
}
if mergedEntries[i].Digest == "" {
- if mergedEntries[i].Size != 0 {
- return nil, fmt.Errorf("missing digest for %q", mergedEntries[i].Name)
- }
- continue
+ return nil, fmt.Errorf("missing digest for %q", mergedEntries[i].Name)
}
digest, err := digest.Parse(mergedEntries[i].Digest)
if err != nil {
@@ -1542,6 +1540,13 @@ func (c *chunkedDiffer) ApplyDiff(dest string, options *archive.TarOptions, diff
if err != nil {
return graphdriver.DriverWithDifferOutput{}, err
}
+
+ // Generate the manifest
+ toc, err := unmarshalToc(c.manifest)
+ if err != nil {
+ return graphdriver.DriverWithDifferOutput{}, err
+ }
+
output := graphdriver.DriverWithDifferOutput{
Differ: c,
TarSplit: c.tarSplit,
@@ -1549,6 +1554,9 @@ func (c *chunkedDiffer) ApplyDiff(dest string, options *archive.TarOptions, diff
bigDataKey: c.manifest,
chunkedLayerDataKey: lcdBigData,
},
+ Artifacts: map[string]interface{}{
+ tocKey: toc,
+ },
TOCDigest: c.contentDigest,
}
@@ -1563,12 +1571,6 @@ func (c *chunkedDiffer) ApplyDiff(dest string, options *archive.TarOptions, diff
// List of OSTree repositories to use for deduplication
ostreeRepos := strings.Split(c.storeOpts.PullOptions["ostree_repos"], ":")
- // Generate the manifest
- toc, err := unmarshalToc(c.manifest)
- if err != nil {
- return output, err
- }
-
whiteoutConverter := archive.GetWhiteoutConverter(options.WhiteoutFormat, options.WhiteoutData)
var missingParts []missingPart
diff --git a/vendor/github.com/klauspost/compress/README.md b/vendor/github.com/klauspost/compress/README.md
index dde75389f1..43de486775 100644
--- a/vendor/github.com/klauspost/compress/README.md
+++ b/vendor/github.com/klauspost/compress/README.md
@@ -16,6 +16,14 @@ This package provides various compression algorithms.
# changelog
+* Sept 19th, 2023 - [v1.17.0](https://github.com/klauspost/compress/releases/tag/v1.17.0)
+ * Add experimental dictionary builder https://github.com/klauspost/compress/pull/853
+ * Add xerial snappy read/writer https://github.com/klauspost/compress/pull/838
+ * flate: Add limited window compression https://github.com/klauspost/compress/pull/843
+ * s2: Do 2 overlapping match checks https://github.com/klauspost/compress/pull/839
+ * flate: Add amd64 assembly matchlen https://github.com/klauspost/compress/pull/837
+ * gzip: Copy bufio.Reader on Reset by @thatguystone in https://github.com/klauspost/compress/pull/860
+
* July 1st, 2023 - [v1.16.7](https://github.com/klauspost/compress/releases/tag/v1.16.7)
* zstd: Fix default level first dictionary encode https://github.com/klauspost/compress/pull/829
* s2: add GetBufferCapacity() method by @GiedriusS in https://github.com/klauspost/compress/pull/832
@@ -646,6 +654,7 @@ Here are other packages of good quality and pure Go (no cgo wrappers or autoconv
* [github.com/ronanh/intcomp](https://github.com/ronanh/intcomp) - Integer compression.
* [github.com/spenczar/fpc](https://github.com/spenczar/fpc) - Float compression.
* [github.com/minio/zipindex](https://github.com/minio/zipindex) - External ZIP directory index.
+* [github.com/ybirader/pzip](https://github.com/ybirader/pzip) - Fast concurrent zip archiver and extractor.
# license
diff --git a/vendor/github.com/klauspost/compress/flate/inflate.go b/vendor/github.com/klauspost/compress/flate/inflate.go
index 414c0bea9f..2f410d64f5 100644
--- a/vendor/github.com/klauspost/compress/flate/inflate.go
+++ b/vendor/github.com/klauspost/compress/flate/inflate.go
@@ -120,8 +120,9 @@ func (h *huffmanDecoder) init(lengths []int) bool {
const sanity = false
if h.chunks == nil {
- h.chunks = &[huffmanNumChunks]uint16{}
+ h.chunks = new([huffmanNumChunks]uint16)
}
+
if h.maxRead != 0 {
*h = huffmanDecoder{chunks: h.chunks, links: h.links}
}
@@ -175,6 +176,7 @@ func (h *huffmanDecoder) init(lengths []int) bool {
}
h.maxRead = min
+
chunks := h.chunks[:]
for i := range chunks {
chunks[i] = 0
@@ -202,8 +204,7 @@ func (h *huffmanDecoder) init(lengths []int) bool {
if cap(h.links[off]) < numLinks {
h.links[off] = make([]uint16, numLinks)
} else {
- links := h.links[off][:0]
- h.links[off] = links[:numLinks]
+ h.links[off] = h.links[off][:numLinks]
}
}
} else {
@@ -277,7 +278,7 @@ func (h *huffmanDecoder) init(lengths []int) bool {
return true
}
-// The actual read interface needed by NewReader.
+// Reader is the actual read interface needed by NewReader.
// If the passed in io.Reader does not also have ReadByte,
// the NewReader will introduce its own buffering.
type Reader interface {
@@ -285,6 +286,18 @@ type Reader interface {
io.ByteReader
}
+type step uint8
+
+const (
+ copyData step = iota + 1
+ nextBlock
+ huffmanBytesBuffer
+ huffmanBytesReader
+ huffmanBufioReader
+ huffmanStringsReader
+ huffmanGenericReader
+)
+
// Decompress state.
type decompressor struct {
// Input source.
@@ -303,7 +316,7 @@ type decompressor struct {
// Next step in the decompression,
// and decompression state.
- step func(*decompressor)
+ step step
stepState int
err error
toRead []byte
@@ -342,7 +355,7 @@ func (f *decompressor) nextBlock() {
// compressed, fixed Huffman tables
f.hl = &fixedHuffmanDecoder
f.hd = nil
- f.huffmanBlockDecoder()()
+ f.huffmanBlockDecoder()
if debugDecode {
fmt.Println("predefinied huffman block")
}
@@ -353,7 +366,7 @@ func (f *decompressor) nextBlock() {
}
f.hl = &f.h1
f.hd = &f.h2
- f.huffmanBlockDecoder()()
+ f.huffmanBlockDecoder()
if debugDecode {
fmt.Println("dynamic huffman block")
}
@@ -379,14 +392,16 @@ func (f *decompressor) Read(b []byte) (int, error) {
if f.err != nil {
return 0, f.err
}
- f.step(f)
+
+ f.doStep()
+
if f.err != nil && len(f.toRead) == 0 {
f.toRead = f.dict.readFlush() // Flush what's left in case of error
}
}
}
-// Support the io.WriteTo interface for io.Copy and friends.
+// WriteTo implements the io.WriteTo interface for io.Copy and friends.
func (f *decompressor) WriteTo(w io.Writer) (int64, error) {
total := int64(0)
flushed := false
@@ -410,7 +425,7 @@ func (f *decompressor) WriteTo(w io.Writer) (int64, error) {
return total, f.err
}
if f.err == nil {
- f.step(f)
+ f.doStep()
}
if len(f.toRead) == 0 && f.err != nil && !flushed {
f.toRead = f.dict.readFlush() // Flush what's left in case of error
@@ -631,7 +646,7 @@ func (f *decompressor) copyData() {
if f.dict.availWrite() == 0 || f.copyLen > 0 {
f.toRead = f.dict.readFlush()
- f.step = (*decompressor).copyData
+ f.step = copyData
return
}
f.finishBlock()
@@ -644,7 +659,28 @@ func (f *decompressor) finishBlock() {
}
f.err = io.EOF
}
- f.step = (*decompressor).nextBlock
+ f.step = nextBlock
+}
+
+func (f *decompressor) doStep() {
+ switch f.step {
+ case copyData:
+ f.copyData()
+ case nextBlock:
+ f.nextBlock()
+ case huffmanBytesBuffer:
+ f.huffmanBytesBuffer()
+ case huffmanBytesReader:
+ f.huffmanBytesReader()
+ case huffmanBufioReader:
+ f.huffmanBufioReader()
+ case huffmanStringsReader:
+ f.huffmanStringsReader()
+ case huffmanGenericReader:
+ f.huffmanGenericReader()
+ default:
+ panic("BUG: unexpected step state")
+ }
}
// noEOF returns err, unless err == io.EOF, in which case it returns io.ErrUnexpectedEOF.
@@ -747,7 +783,7 @@ func (f *decompressor) Reset(r io.Reader, dict []byte) error {
h1: f.h1,
h2: f.h2,
dict: f.dict,
- step: (*decompressor).nextBlock,
+ step: nextBlock,
}
f.dict.init(maxMatchOffset, dict)
return nil
@@ -768,7 +804,7 @@ func NewReader(r io.Reader) io.ReadCloser {
f.r = makeReader(r)
f.bits = new([maxNumLit + maxNumDist]int)
f.codebits = new([numCodes]int)
- f.step = (*decompressor).nextBlock
+ f.step = nextBlock
f.dict.init(maxMatchOffset, nil)
return &f
}
@@ -787,7 +823,7 @@ func NewReaderDict(r io.Reader, dict []byte) io.ReadCloser {
f.r = makeReader(r)
f.bits = new([maxNumLit + maxNumDist]int)
f.codebits = new([numCodes]int)
- f.step = (*decompressor).nextBlock
+ f.step = nextBlock
f.dict.init(maxMatchOffset, dict)
return &f
}
diff --git a/vendor/github.com/klauspost/compress/flate/inflate_gen.go b/vendor/github.com/klauspost/compress/flate/inflate_gen.go
index 61342b6b88..2b2f993f75 100644
--- a/vendor/github.com/klauspost/compress/flate/inflate_gen.go
+++ b/vendor/github.com/klauspost/compress/flate/inflate_gen.go
@@ -85,7 +85,7 @@ readLiteral:
dict.writeByte(byte(v))
if dict.availWrite() == 0 {
f.toRead = dict.readFlush()
- f.step = (*decompressor).huffmanBytesBuffer
+ f.step = huffmanBytesBuffer
f.stepState = stateInit
f.b, f.nb = fb, fnb
return
@@ -251,7 +251,7 @@ copyHistory:
if dict.availWrite() == 0 || f.copyLen > 0 {
f.toRead = dict.readFlush()
- f.step = (*decompressor).huffmanBytesBuffer // We need to continue this work
+ f.step = huffmanBytesBuffer // We need to continue this work
f.stepState = stateDict
f.b, f.nb = fb, fnb
return
@@ -336,7 +336,7 @@ readLiteral:
dict.writeByte(byte(v))
if dict.availWrite() == 0 {
f.toRead = dict.readFlush()
- f.step = (*decompressor).huffmanBytesReader
+ f.step = huffmanBytesReader
f.stepState = stateInit
f.b, f.nb = fb, fnb
return
@@ -502,7 +502,7 @@ copyHistory:
if dict.availWrite() == 0 || f.copyLen > 0 {
f.toRead = dict.readFlush()
- f.step = (*decompressor).huffmanBytesReader // We need to continue this work
+ f.step = huffmanBytesReader // We need to continue this work
f.stepState = stateDict
f.b, f.nb = fb, fnb
return
@@ -587,7 +587,7 @@ readLiteral:
dict.writeByte(byte(v))
if dict.availWrite() == 0 {
f.toRead = dict.readFlush()
- f.step = (*decompressor).huffmanBufioReader
+ f.step = huffmanBufioReader
f.stepState = stateInit
f.b, f.nb = fb, fnb
return
@@ -753,7 +753,7 @@ copyHistory:
if dict.availWrite() == 0 || f.copyLen > 0 {
f.toRead = dict.readFlush()
- f.step = (*decompressor).huffmanBufioReader // We need to continue this work
+ f.step = huffmanBufioReader // We need to continue this work
f.stepState = stateDict
f.b, f.nb = fb, fnb
return
@@ -838,7 +838,7 @@ readLiteral:
dict.writeByte(byte(v))
if dict.availWrite() == 0 {
f.toRead = dict.readFlush()
- f.step = (*decompressor).huffmanStringsReader
+ f.step = huffmanStringsReader
f.stepState = stateInit
f.b, f.nb = fb, fnb
return
@@ -1004,7 +1004,7 @@ copyHistory:
if dict.availWrite() == 0 || f.copyLen > 0 {
f.toRead = dict.readFlush()
- f.step = (*decompressor).huffmanStringsReader // We need to continue this work
+ f.step = huffmanStringsReader // We need to continue this work
f.stepState = stateDict
f.b, f.nb = fb, fnb
return
@@ -1089,7 +1089,7 @@ readLiteral:
dict.writeByte(byte(v))
if dict.availWrite() == 0 {
f.toRead = dict.readFlush()
- f.step = (*decompressor).huffmanGenericReader
+ f.step = huffmanGenericReader
f.stepState = stateInit
f.b, f.nb = fb, fnb
return
@@ -1255,7 +1255,7 @@ copyHistory:
if dict.availWrite() == 0 || f.copyLen > 0 {
f.toRead = dict.readFlush()
- f.step = (*decompressor).huffmanGenericReader // We need to continue this work
+ f.step = huffmanGenericReader // We need to continue this work
f.stepState = stateDict
f.b, f.nb = fb, fnb
return
@@ -1265,19 +1265,19 @@ copyHistory:
// Not reached
}
-func (f *decompressor) huffmanBlockDecoder() func() {
+func (f *decompressor) huffmanBlockDecoder() {
switch f.r.(type) {
case *bytes.Buffer:
- return f.huffmanBytesBuffer
+ f.huffmanBytesBuffer()
case *bytes.Reader:
- return f.huffmanBytesReader
+ f.huffmanBytesReader()
case *bufio.Reader:
- return f.huffmanBufioReader
+ f.huffmanBufioReader()
case *strings.Reader:
- return f.huffmanStringsReader
+ f.huffmanStringsReader()
case Reader:
- return f.huffmanGenericReader
+ f.huffmanGenericReader()
default:
- return f.huffmanGenericReader
+ f.huffmanGenericReader()
}
}
diff --git a/vendor/modules.txt b/vendor/modules.txt
index d443a8e8ac..8a185b3f2c 100644
--- a/vendor/modules.txt
+++ b/vendor/modules.txt
@@ -343,7 +343,7 @@ github.com/containers/psgo/internal/dev
github.com/containers/psgo/internal/host
github.com/containers/psgo/internal/proc
github.com/containers/psgo/internal/process
-# github.com/containers/storage v1.50.3-0.20231005200628-e21971a94abb
+# github.com/containers/storage v1.50.3-0.20231019074621-79aa304201ff
## explicit; go 1.19
github.com/containers/storage
github.com/containers/storage/drivers
@@ -362,6 +362,7 @@ github.com/containers/storage/pkg/archive
github.com/containers/storage/pkg/chrootarchive
github.com/containers/storage/pkg/chunked
github.com/containers/storage/pkg/chunked/compressor
+github.com/containers/storage/pkg/chunked/dump
github.com/containers/storage/pkg/chunked/internal
github.com/containers/storage/pkg/config
github.com/containers/storage/pkg/devicemapper
@@ -688,7 +689,7 @@ github.com/josharian/intern
# github.com/json-iterator/go v1.1.12
## explicit; go 1.12
github.com/json-iterator/go
-# github.com/klauspost/compress v1.17.0
+# github.com/klauspost/compress v1.17.1
## explicit; go 1.18
github.com/klauspost/compress
github.com/klauspost/compress/flate