Update vendor of container/storage

Lots of fixes for issues found by podman.

overlay: propagate errors from mountProgram
utils: root in a userns uses global conf file
Fix handling of additional stores
Correctly check permissions on rootless directory
Fix possible integer overflow on 32bit builds
Evaluate device path for lvm
lockfile test: make concurrent RW test determinisitc
lockfile test: make concurrent read tests deterministic

Signed-off-by: Daniel J Walsh <dwalsh@redhat.com>

vendor.conf

@@ -19,7 +19,7 @@ github.com/containers/image v1.5.1
 github.com/vbauerster/mpb v3.3.4
 github.com/mattn/go-isatty v0.0.4
 github.com/VividCortex/ewma v1.1.1
-github.com/containers/storage v1.12.1
+github.com/containers/storage v1.12.3
 github.com/containers/psgo v1.2
 github.com/coreos/go-systemd v14
 github.com/cri-o/ocicni 0c180f981b27ef6036fa5be29bcb4dd666e406eb

vendor/github.com/containers/storage/containers_ffjson.go

@@ -1,5 +1,5 @@
 // Code generated by ffjson <https://github.com/pquerna/ffjson>. DO NOT EDIT.
-// source: ./containers.go
+// source: containers.go
 
 package storage
 

vendor/github.com/containers/storage/drivers/copy/copy_linux.go

@@ -1,4 +1,4 @@
-// +build linux
+// +build cgo
 
 package copy
 
@@ -153,8 +153,8 @@ func DirCopy(srcDir, dstDir string, copyMode Mode, copyXattrs bool) error {
 
 		isHardlink := false
 
-		switch f.Mode() & os.ModeType {
+		switch mode := f.Mode(); {
-		case 0: // Regular file
+		case mode.IsRegular():
 			id := fileID{dev: stat.Dev, ino: stat.Ino}
 			if copyMode == Hardlink {
 				isHardlink = true
@@ -172,12 +172,12 @@ func DirCopy(srcDir, dstDir string, copyMode Mode, copyXattrs bool) error {
 				copiedFiles[id] = dstPath
 			}
 
-		case os.ModeDir:
+		case mode.IsDir():
 			if err := os.Mkdir(dstPath, f.Mode()); err != nil && !os.IsExist(err) {
 				return err
 			}
 
-		case os.ModeSymlink:
+		case mode&os.ModeSymlink != 0:
 			link, err := os.Readlink(srcPath)
 			if err != nil {
 				return err
@@ -187,14 +187,15 @@ func DirCopy(srcDir, dstDir string, copyMode Mode, copyXattrs bool) error {
 				return err
 			}
 
-		case os.ModeNamedPipe:
+		case mode&os.ModeNamedPipe != 0:
 			fallthrough
-		case os.ModeSocket:
+
+		case mode&os.ModeSocket != 0:
 			if err := unix.Mkfifo(dstPath, stat.Mode); err != nil {
 				return err
 			}
 
-		case os.ModeDevice:
+		case mode&os.ModeDevice != 0:
 			if rsystem.RunningInUserNS() {
 				// cannot create a device if running in user namespace
 				return nil
@@ -204,7 +205,7 @@ func DirCopy(srcDir, dstDir string, copyMode Mode, copyXattrs bool) error {
 			}
 
 		default:
-			return fmt.Errorf("unknown file type for %s", srcPath)
+			return fmt.Errorf("unknown file type with mode %v for %s", mode, srcPath)
 		}
 
 		// Everything below is copying metadata from src to dst. All this metadata

vendor/github.com/containers/storage/drivers/copy/copy_unsupported.go

@@ -0,0 +1,19 @@
+// +build !linux !cgo
+
+package copy
+
+import "github.com/containers/storage/pkg/chrootarchive"
+
+// Mode indicates whether to use hardlink or copy content
+type Mode int
+
+const (
+	// Content creates a new file, and copies the content of the file
+	Content Mode = iota
+)
+
+// DirCopy copies or hardlinks the contents of one directory to another,
+// properly handling soft links
+func DirCopy(srcDir, dstDir string, _ Mode, _ bool) error {
+	return chrootarchive.NewArchiver(nil).CopyWithTar(srcDir, dstDir)
+}

vendor/github.com/containers/storage/drivers/devmapper/device_setup.go

@@ -119,10 +119,17 @@ func checkDevHasFS(dev string) error {
 }
 
 func verifyBlockDevice(dev string, force bool) error {
-	if err := checkDevAvailable(dev); err != nil {
+	realPath, err := filepath.Abs(dev)
+	if err != nil {
+		return errors.Errorf("unable to get absolute path for %s: %s", dev, err)
+	}
+	if realPath, err = filepath.EvalSymlinks(realPath); err != nil {
+		return errors.Errorf("failed to canonicalise path for %s: %s", dev, err)
+	}
+	if err := checkDevAvailable(realPath); err != nil {
 		return err
 	}
-	if err := checkDevInVG(dev); err != nil {
+	if err := checkDevInVG(realPath); err != nil {
 		return err
 	}
 
@@ -130,7 +137,7 @@ func verifyBlockDevice(dev string, force bool) error {
 		return nil
 	}
 
-	if err := checkDevHasFS(dev); err != nil {
+	if err := checkDevHasFS(realPath); err != nil {
 		return err
 	}
 	return nil

vendor/github.com/containers/storage/drivers/overlay/overlay.go

@@ -796,7 +796,17 @@ func (d *Driver) get(id string, disableShifting bool, options graphdriver.MountO
 
 			mountProgram := exec.Command(d.options.mountProgram, "-o", label, target)
 			mountProgram.Dir = d.home
-			return mountProgram.Run()
+			var b bytes.Buffer
+			mountProgram.Stderr = &b
+			err := mountProgram.Run()
+			if err != nil {
+				output := b.String()
+				if output == "" {
+					output = "<stderr empty>"
+				}
+				return errors.Wrapf(err, "using mount program %s: %s", d.options.mountProgram, output)
+			}
+			return nil
 		}
 	} else if len(mountData) > pageSize {
 		//FIXME: We need to figure out to get this to work with additional stores

vendor/github.com/containers/storage/images_ffjson.go

@@ -1,5 +1,5 @@
 // Code generated by ffjson <https://github.com/pquerna/ffjson>. DO NOT EDIT.
-// source: ./images.go
+// source: images.go
 
 package storage
 

vendor/github.com/containers/storage/lockfile.go

@@ -58,8 +58,17 @@ func GetROLockfile(path string) (Locker, error) {
 	return getLockfile(path, true)
 }
 
-// getLockfile is a helper for GetLockfile and GetROLockfile and returns Locker
+// getLockfile returns a Locker object, possibly (depending on the platform)
-// based on the path and read-only property.
+// working inter-process, and associated with the specified path.
+//
+// If ro, the lock is a read-write lock and the returned Locker should correspond to the
+// “lock for reading” (shared) operation; otherwise, the lock is either an exclusive lock,
+// or a read-write lock and Locker should correspond to the “lock for writing” (exclusive) operation.
+//
+// WARNING:
+// - The lock may or MAY NOT be inter-process.
+// - There may or MAY NOT be an actual object on the filesystem created for the specified path.
+// - Even if ro, the lock MAY be exclusive.
 func getLockfile(path string, ro bool) (Locker, error) {
 	lockfilesLock.Lock()
 	defer lockfilesLock.Unlock()
@@ -79,7 +88,7 @@ func getLockfile(path string, ro bool) (Locker, error) {
 		}
 		return locker, nil
 	}
-	locker, err := getLockFile(path, ro) // platform dependent locker
+	locker, err := createLockerForPath(path, ro) // platform-dependent locker
 	if err != nil {
 		return nil, err
 	}

vendor/github.com/containers/storage/lockfile_unix.go

@@ -13,34 +13,6 @@ import (
 	"golang.org/x/sys/unix"
 )
 
-func getLockFile(path string, ro bool) (Locker, error) {
-	var fd int
-	var err error
-	if ro {
-		fd, err = unix.Open(path, os.O_RDONLY, 0)
-	} else {
-		fd, err = unix.Open(path, os.O_RDWR|os.O_CREATE, unix.S_IRUSR|unix.S_IWUSR)
-	}
-	if err != nil {
-		return nil, errors.Wrapf(err, "error opening %q", path)
-	}
-	unix.CloseOnExec(fd)
-
-	locktype := unix.F_WRLCK
-	if ro {
-		locktype = unix.F_RDLCK
-	}
-	return &lockfile{
-		stateMutex: &sync.Mutex{},
-		rwMutex:    &sync.RWMutex{},
-		file:       path,
-		fd:         uintptr(fd),
-		lw:         stringid.GenerateRandomID(),
-		locktype:   int16(locktype),
-		locked:     false,
-		ro:         ro}, nil
-}
-
 type lockfile struct {
 	// rwMutex serializes concurrent reader-writer acquisitions in the same process space
 	rwMutex *sync.RWMutex
@@ -55,6 +27,52 @@ type lockfile struct {
 	ro         bool
 }
 
+// openLock opens the file at path and returns the corresponding file
+// descriptor.  Note that the path is opened read-only when ro is set.  If ro
+// is unset, openLock will open the path read-write and create the file if
+// necessary.
+func openLock(path string, ro bool) (int, error) {
+	if ro {
+		return unix.Open(path, os.O_RDONLY, 0)
+	}
+	return unix.Open(path, os.O_RDWR|os.O_CREATE, unix.S_IRUSR|unix.S_IWUSR)
+}
+
+// createLockerForPath returns a Locker object, possibly (depending on the platform)
+// working inter-process and associated with the specified path.
+//
+// This function will be called at most once for each path value within a single process.
+//
+// If ro, the lock is a read-write lock and the returned Locker should correspond to the
+// “lock for reading” (shared) operation; otherwise, the lock is either an exclusive lock,
+// or a read-write lock and Locker should correspond to the “lock for writing” (exclusive) operation.
+//
+// WARNING:
+// - The lock may or MAY NOT be inter-process.
+// - There may or MAY NOT be an actual object on the filesystem created for the specified path.
+// - Even if ro, the lock MAY be exclusive.
+func createLockerForPath(path string, ro bool) (Locker, error) {
+	// Check if we can open the lock.
+	fd, err := openLock(path, ro)
+	if err != nil {
+		return nil, errors.Wrapf(err, "error opening %q", path)
+	}
+	unix.Close(fd)
+
+	locktype := unix.F_WRLCK
+	if ro {
+		locktype = unix.F_RDLCK
+	}
+	return &lockfile{
+		stateMutex: &sync.Mutex{},
+		rwMutex:    &sync.RWMutex{},
+		file:       path,
+		lw:         stringid.GenerateRandomID(),
+		locktype:   int16(locktype),
+		locked:     false,
+		ro:         ro}, nil
+}
+
 // lock locks the lockfile via FCTNL(2) based on the specified type and
 // command.
 func (l *lockfile) lock(l_type int16) {
@@ -63,7 +81,6 @@ func (l *lockfile) lock(l_type int16) {
 		Whence: int16(os.SEEK_SET),
 		Start:  0,
 		Len:    0,
-		Pid:    int32(os.Getpid()),
 	}
 	switch l_type {
 	case unix.F_RDLCK:
@@ -74,7 +91,16 @@ func (l *lockfile) lock(l_type int16) {
 		panic(fmt.Sprintf("attempted to acquire a file lock of unrecognized type %d", l_type))
 	}
 	l.stateMutex.Lock()
+	defer l.stateMutex.Unlock()
 	if l.counter == 0 {
+		// If we're the first reference on the lock, we need to open the file again.
+		fd, err := openLock(l.file, l.ro)
+		if err != nil {
+			panic(fmt.Sprintf("error opening %q", l.file))
+		}
+		unix.CloseOnExec(fd)
+		l.fd = uintptr(fd)
+
 		// Optimization: only use the (expensive) fcntl syscall when
 		// the counter is 0.  In this case, we're either the first
 		// reader lock or a writer lock.
@@ -85,7 +111,6 @@ func (l *lockfile) lock(l_type int16) {
 	l.locktype = l_type
 	l.locked = true
 	l.counter++
-	l.stateMutex.Unlock()
 }
 
 // Lock locks the lockfile as a writer.  Note that RLock() will be called if
@@ -133,6 +158,8 @@ func (l *lockfile) Unlock() {
 		for unix.FcntlFlock(l.fd, unix.F_SETLKW, &lk) != nil {
 			time.Sleep(10 * time.Millisecond)
 		}
+		// Close the file descriptor on the last unlock.
+		unix.Close(int(l.fd))
 	}
 	if l.locktype == unix.F_RDLCK {
 		l.rwMutex.RUnlock()

vendor/github.com/containers/storage/lockfile_windows.go

@@ -8,7 +8,20 @@ import (
 	"time"
 )
 
-func getLockFile(path string, ro bool) (Locker, error) {
+// createLockerForPath returns a Locker object, possibly (depending on the platform)
+// working inter-process and associated with the specified path.
+//
+// This function will be called at most once for each path value within a single process.
+//
+// If ro, the lock is a read-write lock and the returned Locker should correspond to the
+// “lock for reading” (shared) operation; otherwise, the lock is either an exclusive lock,
+// or a read-write lock and Locker should correspond to the “lock for writing” (exclusive) operation.
+//
+// WARNING:
+// - The lock may or MAY NOT be inter-process.
+// - There may or MAY NOT be an actual object on the filesystem created for the specified path.
+// - Even if ro, the lock MAY be exclusive.
+func createLockerForPath(path string, ro bool) (Locker, error) {
 	return &lockfile{locked: false}, nil
 }
 

vendor/github.com/containers/storage/pkg/idtools/parser.go

@@ -2,6 +2,8 @@ package idtools
 
 import (
 	"fmt"
+	"math"
+	"math/bits"
 	"strconv"
 	"strings"
 )
@@ -31,10 +33,11 @@ func parseTriple(spec []string) (container, host, size uint32, err error) {
 
 // ParseIDMap parses idmap triples from string.
 func ParseIDMap(mapSpec []string, mapSetting string) (idmap []IDMap, err error) {
+	stdErr := fmt.Errorf("error initializing ID mappings: %s setting is malformed", mapSetting)
 	for _, idMapSpec := range mapSpec {
 		idSpec := strings.Fields(strings.Map(nonDigitsToWhitespace, idMapSpec))
 		if len(idSpec)%3 != 0 {
-			return nil, fmt.Errorf("error initializing ID mappings: %s setting is malformed", mapSetting)
+			return nil, stdErr
 		}
 		for i := range idSpec {
 			if i%3 != 0 {
@@ -42,7 +45,11 @@ func ParseIDMap(mapSpec []string, mapSetting string) (idmap []IDMap, err error)
 			}
 			cid, hid, size, err := parseTriple(idSpec[i : i+3])
 			if err != nil {
-				return nil, fmt.Errorf("error initializing ID mappings: %s setting is malformed", mapSetting)
+				return nil, stdErr
+			}
+			// Avoid possible integer overflow on 32bit builds
+			if bits.UintSize == 32 && (cid > math.MaxInt32 || hid > math.MaxInt32 || size > math.MaxInt32) {
+				return nil, stdErr
 			}
 			mapping := IDMap{
 				ContainerID: int(cid),

vendor/github.com/containers/storage/store.go

@@ -460,6 +460,9 @@ type Store interface {
 	// Version returns version information, in the form of key-value pairs, from
 	// the storage package.
 	Version() ([][2]string, error)
+
+	// GetDigestLock returns digest-specific Locker.
+	GetDigestLock(digest.Digest) (Locker, error)
 }
 
 // IDMappingOptions are used for specifying how ID mapping should be set up for
@@ -529,6 +532,7 @@ type store struct {
 	imageStore      ImageStore
 	roImageStores   []ROImageStore
 	containerStore  ContainerStore
+	digestLockRoot  string
 }
 
 // GetStore attempts to find an already-created Store object matching the
@@ -698,9 +702,20 @@ func (s *store) load() error {
 		return err
 	}
 	s.containerStore = rcs
+
+	s.digestLockRoot = filepath.Join(s.runRoot, driverPrefix+"locks")
+	if err := os.MkdirAll(s.digestLockRoot, 0700); err != nil {
+		return err
+	}
+
 	return nil
 }
 
+// GetDigestLock returns a digest-specific Locker.
+func (s *store) GetDigestLock(d digest.Digest) (Locker, error) {
+	return GetLockfile(filepath.Join(s.digestLockRoot, d.String()))
+}
+
 func (s *store) getGraphDriver() (drivers.Driver, error) {
 	if s.graphDriver != nil {
 		return s.graphDriver, nil
@@ -1023,8 +1038,9 @@ func (s *store) imageTopLayerForMapping(image *Image, ristore ROImageStore, crea
 		return reflect.DeepEqual(layer.UIDMap, options.UIDMap) && reflect.DeepEqual(layer.GIDMap, options.GIDMap)
 	}
 	var layer, parentLayer *Layer
+	allStores := append([]ROLayerStore{rlstore}, lstores...)
 	// Locate the image's top layer and its parent, if it has one.
-	for _, s := range append([]ROLayerStore{rlstore}, lstores...) {
+	for _, s := range allStores {
 		store := s
 		if store != rlstore {
 			store.Lock()
@@ -1041,10 +1057,13 @@ func (s *store) imageTopLayerForMapping(image *Image, ristore ROImageStore, crea
 				// We want the layer's parent, too, if it has one.
 				var cParentLayer *Layer
 				if cLayer.Parent != "" {
-					// Its parent should be around here, somewhere.
+					// Its parent should be in one of the stores, somewhere.
-					if cParentLayer, err = store.Get(cLayer.Parent); err != nil {
+					for _, ps := range allStores {
-						// Nope, couldn't find it.  We're not going to be able
+						if cParentLayer, err = ps.Get(cLayer.Parent); err == nil {
-						// to diff this one properly.
+							break
+						}
+					}
+					if cParentLayer == nil {
 						continue
 					}
 				}

vendor/github.com/containers/storage/utils.go

@@ -6,6 +6,7 @@ import (
 	"os/exec"
 	"os/user"
 	"path/filepath"
+	"strconv"
 	"strings"
 
 	"github.com/BurntSushi/toml"
@@ -73,7 +74,7 @@ func GetRootlessRuntimeDir(rootlessUid int) (string, error) {
 	if runtimeDir == "" {
 		tmpDir := fmt.Sprintf("/run/user/%d", rootlessUid)
 		st, err := system.Stat(tmpDir)
-		if err == nil && int(st.UID()) == os.Getuid() && st.Mode() == 0700 {
+		if err == nil && int(st.UID()) == os.Getuid() && st.Mode()&0700 == 0700 && st.Mode()&0066 == 0000 {
 			return tmpDir, nil
 		}
 	}
@@ -158,6 +159,21 @@ func getTomlStorage(storeOptions *StoreOptions) *tomlConfig {
 	return config
 }
 
+func getRootlessUID() int {
+	uidEnv := os.Getenv("_CONTAINERS_ROOTLESS_UID")
+	if uidEnv != "" {
+		u, _ := strconv.Atoi(uidEnv)
+		return u
+	}
+	return os.Geteuid()
+}
+
+// DefaultStoreOptionsAutoDetectUID returns the default storage ops for containers
+func DefaultStoreOptionsAutoDetectUID() (StoreOptions, error) {
+	uid := getRootlessUID()
+	return DefaultStoreOptions(uid != 0, uid)
+}
+
 // DefaultStoreOptions returns the default storage ops for containers
 func DefaultStoreOptions(rootless bool, rootlessUid int) (StoreOptions, error) {
 	var (
@@ -166,14 +182,14 @@ func DefaultStoreOptions(rootless bool, rootlessUid int) (StoreOptions, error) {
 		err                      error
 	)
 	storageOpts := defaultStoreOptions
-	if rootless {
+	if rootless && rootlessUid != 0 {
 		storageOpts, err = getRootlessStorageOpts(rootlessUid)
 		if err != nil {
 			return storageOpts, err
 		}
 	}
 
-	storageConf, err := DefaultConfigFile(rootless)
+	storageConf, err := DefaultConfigFile(rootless && rootlessUid != 0)
 	if err != nil {
 		return storageOpts, err
 	}
@@ -188,7 +204,7 @@ func DefaultStoreOptions(rootless bool, rootlessUid int) (StoreOptions, error) {
 		return storageOpts, errors.Wrapf(err, "cannot stat %s", storageConf)
 	}
 
-	if rootless {
+	if rootless && rootlessUid != 0 {
 		if err == nil {
 			// If the file did not specify a graphroot or runroot,
 			// set sane defaults so we don't try and use root-owned