Update containers/storage to pick up overlay driver fix

New pinned commit is ff8a6d2bf496daf46ab1a153f783a0f6b8762a54

This includes a fix to error reporting with overlayfs, and will
produce more verbose errors when initializing overlayfs fails.

Signed-off-by: Matthew Heon <matthew.heon@gmail.com>

Closes: #546
Approved by: baude

vendor/github.com/containers/storage/pkg/archive/archive.go

@@ -45,6 +45,10 @@ type (
 		// This format will be converted to the standard format on pack
 		// and from the standard format on unpack.
 		WhiteoutFormat WhiteoutFormat
+		// This is additional data to be used by the converter.  It will
+		// not survive a round trip through JSON, so it's primarily
+		// intended for generating archives (i.e., converting writes).
+		WhiteoutData interface{}
 		// When unpacking, specifies whether overwriting a directory with a
 		// non-directory is allowed and vice versa.
 		NoOverwriteDirNonDir bool
@@ -702,7 +706,7 @@ func TarWithOptions(srcPath string, options *TarOptions) (io.ReadCloser, error)
 			compressWriter,
 			options.ChownOpts,
 		)
-		ta.WhiteoutConverter = getWhiteoutConverter(options.WhiteoutFormat)
+		ta.WhiteoutConverter = getWhiteoutConverter(options.WhiteoutFormat, options.WhiteoutData)
 
 		defer func() {
 			// Make sure to check the error on Close.
@@ -860,7 +864,7 @@ func Unpack(decompressedArchive io.Reader, dest string, options *TarOptions) err
 	var dirs []*tar.Header
 	idMappings := idtools.NewIDMappingsFromMaps(options.UIDMaps, options.GIDMaps)
 	rootIDs := idMappings.RootPair()
-	whiteoutConverter := getWhiteoutConverter(options.WhiteoutFormat)
+	whiteoutConverter := getWhiteoutConverter(options.WhiteoutFormat, options.WhiteoutData)
 
 	// Iterate through the files in the archive.
 loop:

vendor/github.com/containers/storage/pkg/archive/archive_linux.go

@@ -5,21 +5,27 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"syscall"
 
 	"github.com/containers/storage/pkg/system"
 	"golang.org/x/sys/unix"
 )
 
-func getWhiteoutConverter(format WhiteoutFormat) tarWhiteoutConverter {
+func getWhiteoutConverter(format WhiteoutFormat, data interface{}) tarWhiteoutConverter {
 	if format == OverlayWhiteoutFormat {
-		return overlayWhiteoutConverter{}
+		if rolayers, ok := data.([]string); ok && len(rolayers) > 0 {
+			return overlayWhiteoutConverter{rolayers: rolayers}
+		}
+		return overlayWhiteoutConverter{rolayers: nil}
 	}
 	return nil
 }
 
-type overlayWhiteoutConverter struct{}
+type overlayWhiteoutConverter struct {
+	rolayers []string
+}
 
-func (overlayWhiteoutConverter) ConvertWrite(hdr *tar.Header, path string, fi os.FileInfo) (wo *tar.Header, err error) {
+func (o overlayWhiteoutConverter) ConvertWrite(hdr *tar.Header, path string, fi os.FileInfo) (wo *tar.Header, err error) {
 	// convert whiteouts to AUFS format
 	if fi.Mode()&os.ModeCharDevice != 0 && hdr.Devmajor == 0 && hdr.Devminor == 0 {
 		// we just rename the file and make it normal
@@ -31,7 +37,7 @@ func (overlayWhiteoutConverter) ConvertWrite(hdr *tar.Header, path string, fi os
 	}
 
 	if fi.Mode()&os.ModeDir != 0 {
-		// convert opaque dirs to AUFS format by writing an empty file with the prefix
+		// convert opaque dirs to AUFS format by writing an empty file with the whiteout prefix
 		opaque, err := system.Lgetxattr(path, "trusted.overlay.opaque")
 		if err != nil {
 			return nil, err
@@ -40,20 +46,64 @@ func (overlayWhiteoutConverter) ConvertWrite(hdr *tar.Header, path string, fi os
 			if hdr.Xattrs != nil {
 				delete(hdr.Xattrs, "trusted.overlay.opaque")
 			}
-
-			// create a header for the whiteout file
-			// it should inherit some properties from the parent, but be a regular file
-			wo = &tar.Header{
-				Typeflag:   tar.TypeReg,
-				Mode:       hdr.Mode & int64(os.ModePerm),
-				Name:       filepath.Join(hdr.Name, WhiteoutOpaqueDir),
-				Size:       0,
-				Uid:        hdr.Uid,
-				Uname:      hdr.Uname,
-				Gid:        hdr.Gid,
-				Gname:      hdr.Gname,
-				AccessTime: hdr.AccessTime,
-				ChangeTime: hdr.ChangeTime,
+			// If there are no lower layers, then it can't have been deleted in this layer.
+			if len(o.rolayers) == 0 {
+				return nil, nil
+			}
+			// At this point, we have a directory that's opaque.  If it appears in one of the lower
+			// layers, then it was newly-created here, so it wasn't also deleted here.
+			for _, rolayer := range o.rolayers {
+				stat, statErr := os.Stat(filepath.Join(rolayer, hdr.Name))
+				if statErr != nil && !os.IsNotExist(statErr) && !isENOTDIR(statErr) {
+					// Not sure what happened here.
+					return nil, statErr
+				}
+				if statErr == nil {
+					if stat.Mode()&os.ModeCharDevice != 0 {
+						// It's a whiteout for this directory, so it can't have been
+						// both deleted and recreated in the layer we're diffing.
+						s := stat.Sys().(*syscall.Stat_t)
+						if major(s.Rdev) == 0 && minor(s.Rdev) == 0 {
+							return nil, nil
+						}
+					}
+					// It's not whiteout, so it was there in the older layer, so we need to
+					// add a whiteout for this item in this layer.
+					// create a header for the whiteout file
+					// it should inherit some properties from the parent, but be a regular file
+					wo = &tar.Header{
+						Typeflag:   tar.TypeReg,
+						Mode:       hdr.Mode & int64(os.ModePerm),
+						Name:       filepath.Join(hdr.Name, WhiteoutOpaqueDir),
+						Size:       0,
+						Uid:        hdr.Uid,
+						Uname:      hdr.Uname,
+						Gid:        hdr.Gid,
+						Gname:      hdr.Gname,
+						AccessTime: hdr.AccessTime,
+						ChangeTime: hdr.ChangeTime,
+					}
+					break
+				}
+				for dir := filepath.Dir(hdr.Name); dir != "" && dir != "." && dir != string(os.PathSeparator); dir = filepath.Dir(dir) {
+					// Check for whiteout for a parent directory in a parent layer.
+					stat, statErr := os.Stat(filepath.Join(rolayer, dir))
+					if statErr != nil && !os.IsNotExist(statErr) && !isENOTDIR(statErr) {
+						// Not sure what happened here.
+						return nil, statErr
+					}
+					if statErr == nil {
+						if stat.Mode()&os.ModeCharDevice != 0 {
+							// If it's whiteout for a parent directory, then the
+							// original directory wasn't inherited into this layer,
+							// so we don't need to emit whiteout for it.
+							s := stat.Sys().(*syscall.Stat_t)
+							if major(s.Rdev) == 0 && minor(s.Rdev) == 0 {
+								return nil, nil
+							}
+						}
+					}
+				}
 			}
 		}
 	}

vendor/github.com/containers/storage/pkg/archive/archive_other.go

@@ -2,6 +2,6 @@
 
 package archive
 
-func getWhiteoutConverter(format WhiteoutFormat) tarWhiteoutConverter {
+func getWhiteoutConverter(format WhiteoutFormat, data interface{}) tarWhiteoutConverter {
 	return nil
 }

vendor/github.com/containers/storage/pkg/archive/changes.go

@@ -81,7 +81,7 @@ func sameFsTimeSpec(a, b syscall.Timespec) bool {
 // Changes walks the path rw and determines changes for the files in the path,
 // with respect to the parent layers
 func Changes(layers []string, rw string) ([]Change, error) {
-	return changes(layers, rw, aufsDeletedFile, aufsMetadataSkip)
+	return changes(layers, rw, aufsDeletedFile, aufsMetadataSkip, aufsWhiteoutPresent)
 }
 
 func aufsMetadataSkip(path string) (skip bool, err error) {
@@ -104,10 +104,35 @@ func aufsDeletedFile(root, path string, fi os.FileInfo) (string, error) {
 	return "", nil
 }
 
+func aufsWhiteoutPresent(root, path string) (bool, error) {
+	f := filepath.Join(filepath.Dir(path), WhiteoutPrefix+filepath.Base(path))
+	_, err := os.Stat(filepath.Join(root, f))
+	if err == nil {
+		return true, nil
+	}
+	if os.IsNotExist(err) || isENOTDIR(err) {
+		return false, nil
+	}
+	return false, err
+}
+
+func isENOTDIR(err error) bool {
+	if err == nil {
+		return false
+	}
+	if perror, ok := err.(*os.PathError); ok {
+		if errno, ok := perror.Err.(syscall.Errno); ok {
+			return errno == syscall.ENOTDIR
+		}
+	}
+	return false
+}
+
 type skipChange func(string) (bool, error)
 type deleteChange func(string, string, os.FileInfo) (string, error)
+type whiteoutChange func(string, string) (bool, error)
 
-func changes(layers []string, rw string, dc deleteChange, sc skipChange) ([]Change, error) {
+func changes(layers []string, rw string, dc deleteChange, sc skipChange, wc whiteoutChange) ([]Change, error) {
 	var (
 		changes     []Change
 		changedDirs = make(map[string]struct{})
@@ -156,7 +181,28 @@ func changes(layers []string, rw string, dc deleteChange, sc skipChange) ([]Chan
 			change.Kind = ChangeAdd
 
 			// ...Unless it already existed in a top layer, in which case, it's a modification
+		layerScan:
 			for _, layer := range layers {
+				if wc != nil {
+					// ...Unless a lower layer also had whiteout for this directory or one of its parents,
+					// in which case, it's new
+					ignore, err := wc(layer, path)
+					if err != nil {
+						return err
+					}
+					if ignore {
+						break layerScan
+					}
+					for dir := filepath.Dir(path); dir != "" && dir != string(os.PathSeparator); dir = filepath.Dir(dir) {
+						ignore, err = wc(layer, dir)
+						if err != nil {
+							return err
+						}
+						if ignore {
+							break layerScan
+						}
+					}
+				}
 				stat, err := os.Stat(filepath.Join(layer, path))
 				if err != nil && !os.IsNotExist(err) {
 					return err
@@ -187,10 +233,15 @@ func changes(layers []string, rw string, dc deleteChange, sc skipChange) ([]Chan
 		}
 		if change.Kind == ChangeAdd || change.Kind == ChangeDelete {
 			parent := filepath.Dir(path)
-			if _, ok := changedDirs[parent]; !ok && parent != "/" {
-				changes = append(changes, Change{Path: parent, Kind: ChangeModify})
-				changedDirs[parent] = struct{}{}
+			tail := []Change{}
+			for parent != "/" {
+				if _, ok := changedDirs[parent]; !ok && parent != "/" {
+					tail = append([]Change{{Path: parent, Kind: ChangeModify}}, tail...)
+					changedDirs[parent] = struct{}{}
+				}
+				parent = filepath.Dir(parent)
 			}
+			changes = append(changes, tail...)
 		}
 
 		// Record change

vendor/github.com/containers/storage/pkg/archive/changes_linux.go

@@ -288,26 +288,96 @@ func clen(n []byte) int {
 // OverlayChanges walks the path rw and determines changes for the files in the path,
 // with respect to the parent layers
 func OverlayChanges(layers []string, rw string) ([]Change, error) {
-	return changes(layers, rw, overlayDeletedFile, nil)
+	dc := func(root, path string, fi os.FileInfo) (string, error) {
+		return overlayDeletedFile(layers, root, path, fi)
+	}
+	return changes(layers, rw, dc, nil, overlayLowerContainsWhiteout)
 }
 
-func overlayDeletedFile(root, path string, fi os.FileInfo) (string, error) {
+func overlayLowerContainsWhiteout(root, path string) (bool, error) {
+	// Whiteout for a file or directory has the same name, but is for a character
+	// device with major/minor of 0/0.
+	stat, err := os.Stat(filepath.Join(root, path))
+	if err != nil && !os.IsNotExist(err) && !isENOTDIR(err) {
+		// Not sure what happened here.
+		return false, err
+	}
+	if err == nil && stat.Mode()&os.ModeCharDevice != 0 {
+		// Check if there's whiteout for the specified item in the specified layer.
+		s := stat.Sys().(*syscall.Stat_t)
+		if major(s.Rdev) == 0 && minor(s.Rdev) == 0 {
+			return true, nil
+		}
+	}
+	return false, nil
+}
+
+func overlayDeletedFile(layers []string, root, path string, fi os.FileInfo) (string, error) {
+	// If it's a whiteout item, then a file or directory with that name is removed by this layer.
 	if fi.Mode()&os.ModeCharDevice != 0 {
 		s := fi.Sys().(*syscall.Stat_t)
 		if major(s.Rdev) == 0 && minor(s.Rdev) == 0 {
 			return path, nil
 		}
 	}
-	if fi.Mode()&os.ModeDir != 0 {
-		opaque, err := system.Lgetxattr(filepath.Join(root, path), "trusted.overlay.opaque")
-		if err != nil {
+	// After this we only need to pay attention to directories.
+	if !fi.IsDir() {
+		return "", nil
+	}
+	// If the directory isn't marked as opaque, then it's just a normal directory.
+	opaque, err := system.Lgetxattr(filepath.Join(root, path), "trusted.overlay.opaque")
+	if err != nil {
+		return "", err
+	}
+	if len(opaque) != 1 || opaque[0] != 'y' {
+		return "", err
+	}
+	// If there are no lower layers, then it can't have been deleted and recreated in this layer.
+	if len(layers) == 0 {
+		return "", err
+	}
+	// At this point, we have a directory that's opaque.  If it appears in one of the lower
+	// layers, then it was newly-created here, so it wasn't also deleted here.
+	for _, layer := range layers {
+		stat, err := os.Stat(filepath.Join(layer, path))
+		if err != nil && !os.IsNotExist(err) && !isENOTDIR(err) {
+			// Not sure what happened here.
 			return "", err
 		}
-		if len(opaque) == 1 && opaque[0] == 'y' {
+		if err == nil {
+			if stat.Mode()&os.ModeCharDevice != 0 {
+				// It's a whiteout for this directory, so it can't have been
+				// deleted in this layer.
+				s := stat.Sys().(*syscall.Stat_t)
+				if major(s.Rdev) == 0 && minor(s.Rdev) == 0 {
+					return "", nil
+				}
+			}
+			// It's not whiteout, so it was there in the older layer, so it has to be
+			// marked as deleted in this layer.
 			return path, nil
 		}
+		for dir := filepath.Dir(path); dir != "" && dir != string(os.PathSeparator); dir = filepath.Dir(dir) {
+			// Check for whiteout for a parent directory.
+			stat, err := os.Stat(filepath.Join(layer, dir))
+			if err != nil && !os.IsNotExist(err) && !isENOTDIR(err) {
+				// Not sure what happened here.
+				return "", err
+			}
+			if err == nil {
+				if stat.Mode()&os.ModeCharDevice != 0 {
+					// If it's whiteout for a parent directory, then the
+					// original directory wasn't inherited into the top layer.
+					s := stat.Sys().(*syscall.Stat_t)
+					if major(s.Rdev) == 0 && minor(s.Rdev) == 0 {
+						return "", nil
+					}
+				}
+			}
+		}
 	}
 
+	// We didn't find the same path in any older layers, so it was new in this one.
 	return "", nil
 
 }