Merge pull request #20359 from containers/renovate/common-image-and-storage-deps

fix(deps): update github.com/containers/storage digest to 79aa304

go.mod

@@ -20,7 +20,7 @@ require (
 	github.com/containers/libhvee v0.4.1-0.20231012183749-e51be96b4854
 	github.com/containers/ocicrypt v1.1.8
 	github.com/containers/psgo v1.8.0
-	github.com/containers/storage v1.50.3-0.20231005200628-e21971a94abb
+	github.com/containers/storage v1.50.3-0.20231019074621-79aa304201ff
 	github.com/coreos/go-systemd/v22 v22.5.0
 	github.com/coreos/stream-metadata-go v0.4.3
 	github.com/crc-org/vfkit v0.1.2-0.20230829083117-09e62065eb6e
@@ -139,7 +139,7 @@ require (
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jinzhu/copier v0.4.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
-	github.com/klauspost/compress v1.17.0 // indirect
+	github.com/klauspost/compress v1.17.1 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.4 // indirect
 	github.com/klauspost/pgzip v1.2.6 // indirect
 	github.com/kr/fs v0.1.0 // indirect

go.sum

@@ -273,8 +273,8 @@ github.com/containers/ocicrypt v1.1.8/go.mod h1:jM362hyBtbwLMWzXQZTlkjKGAQf/BN/L
 github.com/containers/psgo v1.8.0 h1:2loGekmGAxM9ir5OsXWEfGwFxorMPYnc6gEDsGFQvhY=
 github.com/containers/psgo v1.8.0/go.mod h1:T8ZxnX3Ur4RvnhxFJ7t8xJ1F48RhiZB4rSrOaR/qGHc=
 github.com/containers/storage v1.43.0/go.mod h1:uZ147thiIFGdVTjMmIw19knttQnUCl3y9zjreHrg11s=
-github.com/containers/storage v1.50.3-0.20231005200628-e21971a94abb h1:fYLIShJAebUb9l2643IoD9jrj1WEyhux/4/V+2D2CXM=
+github.com/containers/storage v1.50.3-0.20231019074621-79aa304201ff h1:/z85oN77tAw5UUEzymFH73nHIgNtGeL9YednTut9E8M=
-github.com/containers/storage v1.50.3-0.20231005200628-e21971a94abb/go.mod h1:RepNBJN07JUzkeLwmAeRwUstBqisvf8UoW/E6TR1Pak=
+github.com/containers/storage v1.50.3-0.20231019074621-79aa304201ff/go.mod h1:UMgAdUgejmMKJQqeRE+0X/3GP9O6rXRm+E4uS7QrFgk=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/go-iptables v0.4.5/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU=
@@ -680,8 +680,8 @@ github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdY
 github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
 github.com/klauspost/compress v1.15.7/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
 github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
-github.com/klauspost/compress v1.17.0 h1:Rnbp4K9EjcDuVuHtd0dgA4qNuv9yKDYKK1ulpJwgrqM=
+github.com/klauspost/compress v1.17.1 h1:NE3C767s2ak2bweCZo3+rdP4U/HoyVXLv/X9f2gPS5g=
-github.com/klauspost/compress v1.17.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
+github.com/klauspost/compress v1.17.1/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
 github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk=
 github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY=

vendor/github.com/containers/storage/drivers/driver.go

@@ -189,6 +189,9 @@ type DriverWithDifferOutput struct {
 	BigData            map[string][]byte
 	TarSplit           []byte
 	TOCDigest          digest.Digest
+	// Artifacts is a collection of additional artifacts
+	// generated by the differ that the storage driver can use.
+	Artifacts map[string]interface{}
 }
 
 type DifferOutputFormat int

vendor/github.com/containers/storage/drivers/overlay/composefs_notsupported.go

@@ -11,7 +11,7 @@ func composeFsSupported() bool {
 	return false
 }
 
-func generateComposeFsBlob(toc []byte, composefsDir string) error {
+func generateComposeFsBlob(verityDigests map[string]string, toc interface{}, composefsDir string) error {
 	return fmt.Errorf("composefs is not supported")
 }
 
@@ -19,6 +19,6 @@ func mountComposefsBlob(dataDir, mountPoint string) error {
 	return fmt.Errorf("composefs is not supported")
 }
 
-func enableVerityRecursive(path string) error {
+func enableVerityRecursive(path string) (map[string]string, error) {
-	return fmt.Errorf("composefs is not supported")
+	return nil, fmt.Errorf("composefs is not supported")
 }

vendor/github.com/containers/storage/drivers/overlay/composefs_supported.go

@@ -4,7 +4,6 @@
 package overlay
 
 import (
-	"bytes"
 	"encoding/binary"
 	"errors"
 	"fmt"
@@ -16,6 +15,7 @@ import (
 	"syscall"
 	"unsafe"
 
+	"github.com/containers/storage/pkg/chunked/dump"
 	"github.com/containers/storage/pkg/loopback"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/sys/unix"
@@ -29,7 +29,7 @@ var (
 
 func getComposeFsHelper() (string, error) {
 	composeFsHelperOnce.Do(func() {
-		composeFsHelperPath, composeFsHelperErr = exec.LookPath("composefs-from-json")
+		composeFsHelperPath, composeFsHelperErr = exec.LookPath("mkcomposefs")
 	})
 	return composeFsHelperPath, composeFsHelperErr
 }
@@ -53,7 +53,23 @@ func enableVerity(description string, fd int) error {
 	return nil
 }
 
-func enableVerityRecursive(path string) error {
+type verityDigest struct {
+	Fsv unix.FsverityDigest
+	Buf [64]byte
+}
+
+func measureVerity(description string, fd int) (string, error) {
+	var digest verityDigest
+	digest.Fsv.Size = 64
+	_, _, e1 := syscall.Syscall(unix.SYS_IOCTL, uintptr(fd), uintptr(unix.FS_IOC_MEASURE_VERITY), uintptr(unsafe.Pointer(&digest)))
+	if e1 != 0 {
+		return "", fmt.Errorf("failed to measure verity for %q: %w", description, e1)
+	}
+	return fmt.Sprintf("%x", digest.Buf[:digest.Fsv.Size]), nil
+}
+
+func enableVerityRecursive(root string) (map[string]string, error) {
+	digests := make(map[string]string)
 	walkFn := func(path string, d fs.DirEntry, err error) error {
 		if err != nil {
 			return err
@@ -71,24 +87,42 @@ func enableVerityRecursive(path string) error {
 		if err := enableVerity(path, int(f.Fd())); err != nil {
 			return err
 		}
+
+		verity, err := measureVerity(path, int(f.Fd()))
+		if err != nil {
+			return err
+		}
+
+		relPath, err := filepath.Rel(root, path)
+		if err != nil {
+			return err
+		}
+
+		digests[relPath] = verity
 		return nil
 	}
-	return filepath.WalkDir(path, walkFn)
+	err := filepath.WalkDir(root, walkFn)
+	return digests, err
 }
 
 func getComposefsBlob(dataDir string) string {
 	return filepath.Join(dataDir, "composefs.blob")
 }
 
-func generateComposeFsBlob(toc []byte, composefsDir string) error {
+func generateComposeFsBlob(verityDigests map[string]string, toc interface{}, composefsDir string) error {
 	if err := os.MkdirAll(composefsDir, 0o700); err != nil {
 		return err
 	}
 
+	dumpReader, err := dump.GenerateDump(toc, verityDigests)
+	if err != nil {
+		return err
+	}
+
 	destFile := getComposefsBlob(composefsDir)
 	writerJson, err := getComposeFsHelper()
 	if err != nil {
-		return fmt.Errorf("failed to find composefs-from-json: %w", err)
+		return fmt.Errorf("failed to find mkcomposefs: %w", err)
 	}
 
 	fd, err := unix.Openat(unix.AT_FDCWD, destFile, unix.O_WRONLY|unix.O_CREAT|unix.O_TRUNC|unix.O_EXCL|unix.O_CLOEXEC, 0o644)
@@ -109,10 +143,10 @@ func generateComposeFsBlob(toc []byte, composefsDir string) error {
 		// a scope to close outFd before setting fsverity on the read-only fd.
 		defer outFd.Close()
 
-		cmd := exec.Command(writerJson, "--format=erofs", "--out=/proc/self/fd/3", "/proc/self/fd/0")
+		cmd := exec.Command(writerJson, "--from-file", "-", "/proc/self/fd/3")
 		cmd.ExtraFiles = []*os.File{outFd}
 		cmd.Stderr = os.Stderr
-		cmd.Stdin = bytes.NewReader(toc)
+		cmd.Stdin = dumpReader
 		if err := cmd.Run(); err != nil {
 			return fmt.Errorf("failed to convert json to erofs: %w", err)
 		}

vendor/github.com/containers/storage/drivers/overlay/overlay.go

@@ -82,7 +82,7 @@ const (
 	lowerFile  = "lower"
 	maxDepth   = 500
 
-	zstdChunkedManifest = "zstd-chunked-manifest"
+	tocArtifact = "toc"
 
 	// idLength represents the number of random characters
 	// which can be used to create the unique link identifier
@@ -1003,8 +1003,10 @@ func (d *Driver) create(id, parent string, opts *graphdriver.CreateOpts, disable
 		}
 	}
 	if parent != "" {
-		parentBase, parentImageStore, _ := d.dir2(parent)
+		parentBase, parentImageStore, inAdditionalStore := d.dir2(parent)
-		if parentImageStore != "" {
+		// If parentBase path is additional image store, select the image contained in parentBase.
+		// See https://github.com/containers/podman/issues/19748
+		if parentImageStore != "" && !inAdditionalStore {
 			parentBase = parentImageStore
 		}
 		st, err := system.Stat(filepath.Join(parentBase, "diff"))
@@ -1079,12 +1081,13 @@ func (d *Driver) create(id, parent string, opts *graphdriver.CreateOpts, disable
 	}
 
 	if parent != "" {
-		parentDir, parentImageStore, _ := d.dir2(parent)
+		parentBase, parentImageStore, inAdditionalStore := d.dir2(parent)
-		base := parentDir
+		// If parentBase path is additional image store, select the image contained in parentBase.
-		if parentImageStore != "" {
+		// See https://github.com/containers/podman/issues/19748
-			base = parentImageStore
+		if parentImageStore != "" && !inAdditionalStore {
+			parentBase = parentImageStore
 		}
-		st, err := system.Stat(filepath.Join(base, "diff"))
+		st, err := system.Stat(filepath.Join(parentBase, "diff"))
 		if err != nil {
 			return err
 		}
@@ -1526,15 +1529,8 @@ func (d *Driver) get(id string, disableShifting bool, options graphdriver.MountO
 		defer cleanupFunc()
 	}
 
-	composefsLayers := filepath.Join(workDirBase, "composefs-layers")
-	if err := os.MkdirAll(composefsLayers, 0o700); err != nil {
-		return "", err
-	}
-
 	skipIDMappingLayers := make(map[string]string)
 
-	composeFsLayers := []string{}
-
 	composefsMounts := []string{}
 	defer func() {
 		for _, m := range composefsMounts {
@@ -1542,6 +1538,8 @@ func (d *Driver) get(id string, disableShifting bool, options graphdriver.MountO
 		}
 	}()
 
+	composeFsLayers := []string{}
+	composeFsLayersDir := filepath.Join(workDirBase, "composefs-layers")
 	maybeAddComposefsMount := func(lowerID string, i int, readWrite bool) (string, error) {
 		composefsBlob := d.getComposefsData(lowerID)
 		_, err = os.Stat(composefsBlob)
@@ -1557,7 +1555,7 @@ func (d *Driver) get(id string, disableShifting bool, options graphdriver.MountO
 			return "", fmt.Errorf("cannot mount a composefs layer as writeable")
 		}
 
-		dest := filepath.Join(composefsLayers, fmt.Sprintf("%d", i))
+		dest := filepath.Join(composeFsLayersDir, fmt.Sprintf("%d", i))
 		if err := os.MkdirAll(dest, 0o700); err != nil {
 			return "", err
 		}
@@ -2110,11 +2108,12 @@ func (d *Driver) ApplyDiffFromStagingDirectory(id, parent, stagingDirectory stri
 	if d.useComposeFs() {
 		// FIXME: move this logic into the differ so we don't have to open
 		// the file twice.
-		if err := enableVerityRecursive(stagingDirectory); err != nil && !errors.Is(err, unix.ENOTSUP) && !errors.Is(err, unix.ENOTTY) {
+		verityDigests, err := enableVerityRecursive(stagingDirectory)
+		if err != nil && !errors.Is(err, unix.ENOTSUP) && !errors.Is(err, unix.ENOTTY) {
 			logrus.Warningf("%s", err)
 		}
-		toc := diffOutput.BigData[zstdChunkedManifest]
+		toc := diffOutput.Artifacts[tocArtifact]
-		if err := generateComposeFsBlob(toc, d.getComposefsData(id)); err != nil {
+		if err := generateComposeFsBlob(verityDigests, toc, d.getComposefsData(id)); err != nil {
 			return err
 		}
 	}

vendor/github.com/containers/storage/pkg/chunked/cache_linux.go

@@ -26,6 +26,8 @@ import (
 const (
 	cacheKey     = "chunked-manifest-cache"
 	cacheVersion = 1
+
+	digestSha256Empty = "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
 )
 
 type metadata struct {
@@ -650,6 +652,9 @@ func unmarshalToc(manifest []byte) (*internal.TOC, error) {
 					iter.Skip()
 				}
 			}
+			if m.Type == TypeReg && m.Size == 0 && m.Digest == "" {
+				m.Digest = digestSha256Empty
+			}
 			toc.Entries = append(toc.Entries, m)
 		}
 		break

vendor/github.com/containers/storage/pkg/chunked/dump/dump.go

@@ -0,0 +1,230 @@
+package dump
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"strings"
+	"time"
+	"unicode"
+
+	"github.com/containers/storage/pkg/chunked/internal"
+	"golang.org/x/sys/unix"
+)
+
+const (
+	ESCAPE_STANDARD = 0
+	NOESCAPE_SPACE  = 1 << iota
+	ESCAPE_EQUAL
+	ESCAPE_LONE_DASH
+)
+
+func escaped(val string, escape int) string {
+	noescapeSpace := escape&NOESCAPE_SPACE != 0
+	escapeEqual := escape&ESCAPE_EQUAL != 0
+	escapeLoneDash := escape&ESCAPE_LONE_DASH != 0
+
+	length := len(val)
+
+	if escapeLoneDash && val == "-" {
+		return fmt.Sprintf("\\x%.2x", val[0])
+	}
+
+	var result string
+	for i := 0; i < length; i++ {
+		c := val[i]
+		hexEscape := false
+		var special string
+
+		switch c {
+		case '\\':
+			special = "\\\\"
+		case '\n':
+			special = "\\n"
+		case '\r':
+			special = "\\r"
+		case '\t':
+			special = "\\t"
+		case '=':
+			hexEscape = escapeEqual
+		default:
+			if noescapeSpace {
+				hexEscape = !unicode.IsPrint(rune(c))
+			} else {
+				hexEscape = !unicode.IsGraphic(rune(c))
+			}
+		}
+
+		if special != "" {
+			result += special
+		} else if hexEscape {
+			result += fmt.Sprintf("\\x%.2x", c)
+		} else {
+			result += string(c)
+		}
+	}
+	return result
+}
+
+func escapedOptional(val string, escape int) string {
+	if val == "" {
+		return "-"
+	}
+	return escaped(val, escape)
+}
+
+func getStMode(mode uint32, typ string) (uint32, error) {
+	switch typ {
+	case internal.TypeReg, internal.TypeLink:
+		mode |= unix.S_IFREG
+	case internal.TypeChar:
+		mode |= unix.S_IFCHR
+	case internal.TypeBlock:
+		mode |= unix.S_IFBLK
+	case internal.TypeDir:
+		mode |= unix.S_IFDIR
+	case internal.TypeFifo:
+		mode |= unix.S_IFIFO
+	case internal.TypeSymlink:
+		mode |= unix.S_IFLNK
+	default:
+		return 0, fmt.Errorf("unknown type %s", typ)
+	}
+	return mode, nil
+}
+
+func dumpNode(out io.Writer, links map[string]int, verityDigests map[string]string, entry *internal.FileMetadata) error {
+	path := entry.Name
+	if path == "" {
+		path = "/"
+	} else if path[0] != '/' {
+		path = "/" + path
+	}
+
+	if _, err := fmt.Fprint(out, escaped(path, ESCAPE_STANDARD)); err != nil {
+		return err
+	}
+
+	nlinks := links[entry.Name] + links[entry.Linkname] + 1
+	link := ""
+	if entry.Type == internal.TypeLink {
+		link = "@"
+	}
+
+	rdev := unix.Mkdev(uint32(entry.Devmajor), uint32(entry.Devminor))
+
+	entryTime := entry.ModTime
+	if entryTime == nil {
+		t := time.Unix(0, 0)
+		entryTime = &t
+	}
+
+	mode, err := getStMode(uint32(entry.Mode), entry.Type)
+	if err != nil {
+		return err
+	}
+
+	if _, err := fmt.Fprintf(out, " %d %s%o %d %d %d %d %d.%d ", entry.Size,
+		link, mode,
+		nlinks, entry.UID, entry.GID, rdev,
+		entryTime.Unix(), entryTime.Nanosecond()); err != nil {
+		return err
+	}
+
+	var payload string
+	if entry.Linkname != "" {
+		payload = entry.Linkname
+		if entry.Type == internal.TypeLink && payload[0] != '/' {
+			payload = "/" + payload
+		}
+	} else {
+		if len(entry.Digest) > 10 {
+			d := strings.Replace(entry.Digest, "sha256:", "", 1)
+			payload = d[:2] + "/" + d[2:]
+		}
+	}
+
+	if _, err := fmt.Fprintf(out, escapedOptional(payload, ESCAPE_LONE_DASH)); err != nil {
+		return err
+	}
+
+	/* inline content.  */
+	if _, err := fmt.Fprint(out, " -"); err != nil {
+		return err
+	}
+
+	/* store the digest.  */
+	if _, err := fmt.Fprint(out, " "); err != nil {
+		return err
+	}
+	digest := verityDigests[payload]
+	if _, err := fmt.Fprintf(out, escapedOptional(digest, ESCAPE_LONE_DASH)); err != nil {
+		return err
+	}
+
+	for k, v := range entry.Xattrs {
+		name := escaped(k, ESCAPE_EQUAL)
+		value := escaped(v, ESCAPE_EQUAL)
+
+		if _, err := fmt.Fprintf(out, " %s=%s", name, value); err != nil {
+			return err
+		}
+	}
+	if _, err := fmt.Fprint(out, "\n"); err != nil {
+		return err
+	}
+	return nil
+}
+
+// GenerateDump generates a dump of the TOC in the same format as `composefs-info dump`
+func GenerateDump(tocI interface{}, verityDigests map[string]string) (io.Reader, error) {
+	toc, ok := tocI.(*internal.TOC)
+	if !ok {
+		return nil, fmt.Errorf("invalid TOC type")
+	}
+	pipeR, pipeW := io.Pipe()
+	go func() {
+		closed := false
+		w := bufio.NewWriter(pipeW)
+		defer func() {
+			if !closed {
+				w.Flush()
+				pipeW.Close()
+			}
+		}()
+
+		links := make(map[string]int)
+		for _, e := range toc.Entries {
+			if e.Linkname == "" {
+				continue
+			}
+			links[e.Linkname] = links[e.Linkname] + 1
+		}
+
+		if len(toc.Entries) == 0 || (toc.Entries[0].Name != "" && toc.Entries[0].Name != "/") {
+			root := &internal.FileMetadata{
+				Name: "/",
+				Type: internal.TypeDir,
+				Mode: 0o755,
+			}
+
+			if err := dumpNode(w, links, verityDigests, root); err != nil {
+				pipeW.CloseWithError(err)
+				closed = true
+				return
+			}
+		}
+
+		for _, e := range toc.Entries {
+			if e.Type == internal.TypeChunk {
+				continue
+			}
+			if err := dumpNode(w, links, verityDigests, &e); err != nil {
+				pipeW.CloseWithError(err)
+				closed = true
+				return
+			}
+		}
+	}()
+	return pipeR, nil
+}

vendor/github.com/containers/storage/pkg/chunked/storage_linux.go

@@ -45,6 +45,7 @@ const (
 	bigDataKey              = "zstd-chunked-manifest"
 	chunkedData             = "zstd-chunked-data"
 	chunkedLayerDataKey     = "zstd-chunked-layer-data"
+	tocKey                  = "toc"
 
 	fileTypeZstdChunked = iota
 	fileTypeEstargz
@@ -1470,11 +1471,8 @@ func makeEntriesFlat(mergedEntries []internal.FileMetadata) ([]internal.FileMeta
 			continue
 		}
 		if mergedEntries[i].Digest == "" {
-			if mergedEntries[i].Size != 0 {
 			return nil, fmt.Errorf("missing digest for %q", mergedEntries[i].Name)
 		}
-			continue
-		}
 		digest, err := digest.Parse(mergedEntries[i].Digest)
 		if err != nil {
 			return nil, err
@@ -1542,6 +1540,13 @@ func (c *chunkedDiffer) ApplyDiff(dest string, options *archive.TarOptions, diff
 	if err != nil {
 		return graphdriver.DriverWithDifferOutput{}, err
 	}
+
+	// Generate the manifest
+	toc, err := unmarshalToc(c.manifest)
+	if err != nil {
+		return graphdriver.DriverWithDifferOutput{}, err
+	}
+
 	output := graphdriver.DriverWithDifferOutput{
 		Differ:   c,
 		TarSplit: c.tarSplit,
@@ -1549,6 +1554,9 @@ func (c *chunkedDiffer) ApplyDiff(dest string, options *archive.TarOptions, diff
 			bigDataKey:          c.manifest,
 			chunkedLayerDataKey: lcdBigData,
 		},
+		Artifacts: map[string]interface{}{
+			tocKey: toc,
+		},
 		TOCDigest: c.contentDigest,
 	}
 
@@ -1563,12 +1571,6 @@ func (c *chunkedDiffer) ApplyDiff(dest string, options *archive.TarOptions, diff
 	// List of OSTree repositories to use for deduplication
 	ostreeRepos := strings.Split(c.storeOpts.PullOptions["ostree_repos"], ":")
 
-	// Generate the manifest
-	toc, err := unmarshalToc(c.manifest)
-	if err != nil {
-		return output, err
-	}
-
 	whiteoutConverter := archive.GetWhiteoutConverter(options.WhiteoutFormat, options.WhiteoutData)
 
 	var missingParts []missingPart

vendor/github.com/klauspost/compress/README.md

@@ -16,6 +16,14 @@ This package provides various compression algorithms.
 
 # changelog
 
+* Sept 19th, 2023 - [v1.17.0](https://github.com/klauspost/compress/releases/tag/v1.17.0)
+	* Add experimental dictionary builder  https://github.com/klauspost/compress/pull/853
+	* Add xerial snappy read/writer https://github.com/klauspost/compress/pull/838
+	* flate: Add limited window compression https://github.com/klauspost/compress/pull/843
+	* s2: Do 2 overlapping match checks https://github.com/klauspost/compress/pull/839
+	* flate: Add amd64 assembly matchlen https://github.com/klauspost/compress/pull/837
+	* gzip: Copy bufio.Reader on Reset by @thatguystone in https://github.com/klauspost/compress/pull/860
+   
 * July 1st, 2023 - [v1.16.7](https://github.com/klauspost/compress/releases/tag/v1.16.7)
 	* zstd: Fix default level first dictionary encode https://github.com/klauspost/compress/pull/829
 	* s2: add GetBufferCapacity() method by @GiedriusS in https://github.com/klauspost/compress/pull/832
@@ -646,6 +654,7 @@ Here are other packages of good quality and pure Go (no cgo wrappers or autoconv
 * [github.com/ronanh/intcomp](https://github.com/ronanh/intcomp) - Integer compression.
 * [github.com/spenczar/fpc](https://github.com/spenczar/fpc) - Float compression.
 * [github.com/minio/zipindex](https://github.com/minio/zipindex) - External ZIP directory index.
+* [github.com/ybirader/pzip](https://github.com/ybirader/pzip) - Fast concurrent zip archiver and extractor.
 
 # license
 

vendor/github.com/klauspost/compress/flate/inflate.go

@@ -120,8 +120,9 @@ func (h *huffmanDecoder) init(lengths []int) bool {
 	const sanity = false
 
 	if h.chunks == nil {
-		h.chunks = &[huffmanNumChunks]uint16{}
+		h.chunks = new([huffmanNumChunks]uint16)
 	}
+
 	if h.maxRead != 0 {
 		*h = huffmanDecoder{chunks: h.chunks, links: h.links}
 	}
@@ -175,6 +176,7 @@ func (h *huffmanDecoder) init(lengths []int) bool {
 	}
 
 	h.maxRead = min
+
 	chunks := h.chunks[:]
 	for i := range chunks {
 		chunks[i] = 0
@@ -202,8 +204,7 @@ func (h *huffmanDecoder) init(lengths []int) bool {
 			if cap(h.links[off]) < numLinks {
 				h.links[off] = make([]uint16, numLinks)
 			} else {
-				links := h.links[off][:0]
+				h.links[off] = h.links[off][:numLinks]
-				h.links[off] = links[:numLinks]
 			}
 		}
 	} else {
@@ -277,7 +278,7 @@ func (h *huffmanDecoder) init(lengths []int) bool {
 	return true
 }
 
-// The actual read interface needed by NewReader.
+// Reader is the actual read interface needed by NewReader.
 // If the passed in io.Reader does not also have ReadByte,
 // the NewReader will introduce its own buffering.
 type Reader interface {
@@ -285,6 +286,18 @@ type Reader interface {
 	io.ByteReader
 }
 
+type step uint8
+
+const (
+	copyData step = iota + 1
+	nextBlock
+	huffmanBytesBuffer
+	huffmanBytesReader
+	huffmanBufioReader
+	huffmanStringsReader
+	huffmanGenericReader
+)
+
 // Decompress state.
 type decompressor struct {
 	// Input source.
@@ -303,7 +316,7 @@ type decompressor struct {
 
 	// Next step in the decompression,
 	// and decompression state.
-	step      func(*decompressor)
+	step      step
 	stepState int
 	err       error
 	toRead    []byte
@@ -342,7 +355,7 @@ func (f *decompressor) nextBlock() {
 		// compressed, fixed Huffman tables
 		f.hl = &fixedHuffmanDecoder
 		f.hd = nil
-		f.huffmanBlockDecoder()()
+		f.huffmanBlockDecoder()
 		if debugDecode {
 			fmt.Println("predefinied huffman block")
 		}
@@ -353,7 +366,7 @@ func (f *decompressor) nextBlock() {
 		}
 		f.hl = &f.h1
 		f.hd = &f.h2
-		f.huffmanBlockDecoder()()
+		f.huffmanBlockDecoder()
 		if debugDecode {
 			fmt.Println("dynamic huffman block")
 		}
@@ -379,14 +392,16 @@ func (f *decompressor) Read(b []byte) (int, error) {
 		if f.err != nil {
 			return 0, f.err
 		}
-		f.step(f)
+
+		f.doStep()
+
 		if f.err != nil && len(f.toRead) == 0 {
 			f.toRead = f.dict.readFlush() // Flush what's left in case of error
 		}
 	}
 }
 
-// Support the io.WriteTo interface for io.Copy and friends.
+// WriteTo implements the io.WriteTo interface for io.Copy and friends.
 func (f *decompressor) WriteTo(w io.Writer) (int64, error) {
 	total := int64(0)
 	flushed := false
@@ -410,7 +425,7 @@ func (f *decompressor) WriteTo(w io.Writer) (int64, error) {
 			return total, f.err
 		}
 		if f.err == nil {
-			f.step(f)
+			f.doStep()
 		}
 		if len(f.toRead) == 0 && f.err != nil && !flushed {
 			f.toRead = f.dict.readFlush() // Flush what's left in case of error
@@ -631,7 +646,7 @@ func (f *decompressor) copyData() {
 
 	if f.dict.availWrite() == 0 || f.copyLen > 0 {
 		f.toRead = f.dict.readFlush()
-		f.step = (*decompressor).copyData
+		f.step = copyData
 		return
 	}
 	f.finishBlock()
@@ -644,7 +659,28 @@ func (f *decompressor) finishBlock() {
 		}
 		f.err = io.EOF
 	}
-	f.step = (*decompressor).nextBlock
+	f.step = nextBlock
+}
+
+func (f *decompressor) doStep() {
+	switch f.step {
+	case copyData:
+		f.copyData()
+	case nextBlock:
+		f.nextBlock()
+	case huffmanBytesBuffer:
+		f.huffmanBytesBuffer()
+	case huffmanBytesReader:
+		f.huffmanBytesReader()
+	case huffmanBufioReader:
+		f.huffmanBufioReader()
+	case huffmanStringsReader:
+		f.huffmanStringsReader()
+	case huffmanGenericReader:
+		f.huffmanGenericReader()
+	default:
+		panic("BUG: unexpected step state")
+	}
 }
 
 // noEOF returns err, unless err == io.EOF, in which case it returns io.ErrUnexpectedEOF.
@@ -747,7 +783,7 @@ func (f *decompressor) Reset(r io.Reader, dict []byte) error {
 		h1:       f.h1,
 		h2:       f.h2,
 		dict:     f.dict,
-		step:     (*decompressor).nextBlock,
+		step:     nextBlock,
 	}
 	f.dict.init(maxMatchOffset, dict)
 	return nil
@@ -768,7 +804,7 @@ func NewReader(r io.Reader) io.ReadCloser {
 	f.r = makeReader(r)
 	f.bits = new([maxNumLit + maxNumDist]int)
 	f.codebits = new([numCodes]int)
-	f.step = (*decompressor).nextBlock
+	f.step = nextBlock
 	f.dict.init(maxMatchOffset, nil)
 	return &f
 }
@@ -787,7 +823,7 @@ func NewReaderDict(r io.Reader, dict []byte) io.ReadCloser {
 	f.r = makeReader(r)
 	f.bits = new([maxNumLit + maxNumDist]int)
 	f.codebits = new([numCodes]int)
-	f.step = (*decompressor).nextBlock
+	f.step = nextBlock
 	f.dict.init(maxMatchOffset, dict)
 	return &f
 }

vendor/github.com/klauspost/compress/flate/inflate_gen.go

@@ -85,7 +85,7 @@ readLiteral:
 			dict.writeByte(byte(v))
 			if dict.availWrite() == 0 {
 				f.toRead = dict.readFlush()
-				f.step = (*decompressor).huffmanBytesBuffer
+				f.step = huffmanBytesBuffer
 				f.stepState = stateInit
 				f.b, f.nb = fb, fnb
 				return
@@ -251,7 +251,7 @@ copyHistory:
 
 		if dict.availWrite() == 0 || f.copyLen > 0 {
 			f.toRead = dict.readFlush()
-			f.step = (*decompressor).huffmanBytesBuffer // We need to continue this work
+			f.step = huffmanBytesBuffer // We need to continue this work
 			f.stepState = stateDict
 			f.b, f.nb = fb, fnb
 			return
@@ -336,7 +336,7 @@ readLiteral:
 			dict.writeByte(byte(v))
 			if dict.availWrite() == 0 {
 				f.toRead = dict.readFlush()
-				f.step = (*decompressor).huffmanBytesReader
+				f.step = huffmanBytesReader
 				f.stepState = stateInit
 				f.b, f.nb = fb, fnb
 				return
@@ -502,7 +502,7 @@ copyHistory:
 
 		if dict.availWrite() == 0 || f.copyLen > 0 {
 			f.toRead = dict.readFlush()
-			f.step = (*decompressor).huffmanBytesReader // We need to continue this work
+			f.step = huffmanBytesReader // We need to continue this work
 			f.stepState = stateDict
 			f.b, f.nb = fb, fnb
 			return
@@ -587,7 +587,7 @@ readLiteral:
 			dict.writeByte(byte(v))
 			if dict.availWrite() == 0 {
 				f.toRead = dict.readFlush()
-				f.step = (*decompressor).huffmanBufioReader
+				f.step = huffmanBufioReader
 				f.stepState = stateInit
 				f.b, f.nb = fb, fnb
 				return
@@ -753,7 +753,7 @@ copyHistory:
 
 		if dict.availWrite() == 0 || f.copyLen > 0 {
 			f.toRead = dict.readFlush()
-			f.step = (*decompressor).huffmanBufioReader // We need to continue this work
+			f.step = huffmanBufioReader // We need to continue this work
 			f.stepState = stateDict
 			f.b, f.nb = fb, fnb
 			return
@@ -838,7 +838,7 @@ readLiteral:
 			dict.writeByte(byte(v))
 			if dict.availWrite() == 0 {
 				f.toRead = dict.readFlush()
-				f.step = (*decompressor).huffmanStringsReader
+				f.step = huffmanStringsReader
 				f.stepState = stateInit
 				f.b, f.nb = fb, fnb
 				return
@@ -1004,7 +1004,7 @@ copyHistory:
 
 		if dict.availWrite() == 0 || f.copyLen > 0 {
 			f.toRead = dict.readFlush()
-			f.step = (*decompressor).huffmanStringsReader // We need to continue this work
+			f.step = huffmanStringsReader // We need to continue this work
 			f.stepState = stateDict
 			f.b, f.nb = fb, fnb
 			return
@@ -1089,7 +1089,7 @@ readLiteral:
 			dict.writeByte(byte(v))
 			if dict.availWrite() == 0 {
 				f.toRead = dict.readFlush()
-				f.step = (*decompressor).huffmanGenericReader
+				f.step = huffmanGenericReader
 				f.stepState = stateInit
 				f.b, f.nb = fb, fnb
 				return
@@ -1255,7 +1255,7 @@ copyHistory:
 
 		if dict.availWrite() == 0 || f.copyLen > 0 {
 			f.toRead = dict.readFlush()
-			f.step = (*decompressor).huffmanGenericReader // We need to continue this work
+			f.step = huffmanGenericReader // We need to continue this work
 			f.stepState = stateDict
 			f.b, f.nb = fb, fnb
 			return
@@ -1265,19 +1265,19 @@ copyHistory:
 	// Not reached
 }
 
-func (f *decompressor) huffmanBlockDecoder() func() {
+func (f *decompressor) huffmanBlockDecoder() {
 	switch f.r.(type) {
 	case *bytes.Buffer:
-		return f.huffmanBytesBuffer
+		f.huffmanBytesBuffer()
 	case *bytes.Reader:
-		return f.huffmanBytesReader
+		f.huffmanBytesReader()
 	case *bufio.Reader:
-		return f.huffmanBufioReader
+		f.huffmanBufioReader()
 	case *strings.Reader:
-		return f.huffmanStringsReader
+		f.huffmanStringsReader()
 	case Reader:
-		return f.huffmanGenericReader
+		f.huffmanGenericReader()
 	default:
-		return f.huffmanGenericReader
+		f.huffmanGenericReader()
 	}
 }

vendor/modules.txt

@@ -343,7 +343,7 @@ github.com/containers/psgo/internal/dev
 github.com/containers/psgo/internal/host
 github.com/containers/psgo/internal/proc
 github.com/containers/psgo/internal/process
-# github.com/containers/storage v1.50.3-0.20231005200628-e21971a94abb
+# github.com/containers/storage v1.50.3-0.20231019074621-79aa304201ff
 ## explicit; go 1.19
 github.com/containers/storage
 github.com/containers/storage/drivers
@@ -362,6 +362,7 @@ github.com/containers/storage/pkg/archive
 github.com/containers/storage/pkg/chrootarchive
 github.com/containers/storage/pkg/chunked
 github.com/containers/storage/pkg/chunked/compressor
+github.com/containers/storage/pkg/chunked/dump
 github.com/containers/storage/pkg/chunked/internal
 github.com/containers/storage/pkg/config
 github.com/containers/storage/pkg/devicemapper
@@ -688,7 +689,7 @@ github.com/josharian/intern
 # github.com/json-iterator/go v1.1.12
 ## explicit; go 1.12
 github.com/json-iterator/go
-# github.com/klauspost/compress v1.17.0
+# github.com/klauspost/compress v1.17.1
 ## explicit; go 1.18
 github.com/klauspost/compress
 github.com/klauspost/compress/flate