Vendor in latest containers/storage and containers/image

Update container/image to address a commit error when copying layers and metadata.
This change may require users to recreate containers.

container/storage added some new lock protection to prevent possible deadlock and
data corruption.

Signed-off-by: Daniel J Walsh <dwalsh@redhat.com>

Closes: #1381
Approved by: mheon

vendor.conf

@@ -10,8 +10,8 @@ github.com/containerd/cgroups 77e628511d924b13a77cebdc73b757a47f6d751b
 github.com/containerd/continuity master
 github.com/containernetworking/cni v0.7.0-alpha1
 github.com/containernetworking/plugins 1562a1e60ed101aacc5e08ed9dbeba8e9f3d4ec1
-github.com/containers/image 134f99bed228d6297dc01d152804f6f09f185418
-github.com/containers/storage 17c7d1fee5603ccf6dd97edc14162fc1510e7e23
+github.com/containers/image 5df44e095ed826fbe2beeaabb329c749d7d6c3b6
+github.com/containers/storage d0cb0107646058ad96ce90630fee2bd5709ee637
 github.com/containers/psgo 5dde6da0bc8831b35243a847625bcf18183bd1ee
 github.com/coreos/go-systemd v14
 github.com/cri-o/ocicni master

vendor/github.com/containers/image/README.md

@@ -25,7 +25,7 @@ them as necessary, and to sign and verify images.
 The containers/image project is only a library with no user interface;
 you can either incorporate it into your Go programs, or use the `skopeo` tool:
 
-The [skopeo](https://github.com/projectatomic/skopeo) tool uses the
+The [skopeo](https://github.com/containers/skopeo) tool uses the
 containers/image library and takes advantage of many of its features,
 e.g. `skopeo copy` exposes the `containers/image/copy.Image` functionality.
 
@@ -42,7 +42,7 @@ What this project tests against dependencies-wise is located
 ## Building
 
 If you want to see what the library can do, or an example of how it is called,
-consider starting with the [skopeo](https://github.com/projectatomic/skopeo) tool
+consider starting with the [skopeo](https://github.com/containers/skopeo) tool
 instead.
 
 To integrate this library into your project, put it into `$GOPATH` or use
@@ -73,7 +73,9 @@ When developing this library, please use `make` (or `make … BUILDTAGS=…`) to
 
 ## License
 
-ASL 2.0
+Apache License 2.0
+
+SPDX-License-Identifier: Apache-2.0
 
 ## Contact
 

vendor/github.com/containers/image/copy/copy.go

@@ -71,7 +71,6 @@ func (d *digestingReader) Read(p []byte) (int, error) {
 // copier allows us to keep track of diffID values for blobs, and other
 // data shared across one or more images in a possible manifest list.
 type copier struct {
-	copiedBlobs      map[digest.Digest]digest.Digest
 	cachedDiffIDs    map[digest.Digest]digest.Digest
 	dest             types.ImageDestination
 	rawSource        types.ImageSource
@@ -141,7 +140,6 @@ func Image(ctx context.Context, policyContext *signature.PolicyContext, destRef,
 	}()
 
 	c := &copier{
-		copiedBlobs:      make(map[digest.Digest]digest.Digest),
 		cachedDiffIDs:    make(map[digest.Digest]digest.Digest),
 		dest:             dest,
 		rawSource:        rawSource,

vendor/github.com/containers/image/docker/docker_client.go

@@ -462,6 +462,7 @@ func (c *dockerClient) getBearerToken(ctx context.Context, realm, service, scope
 	if c.username != "" && c.password != "" {
 		authReq.SetBasicAuth(c.username, c.password)
 	}
+	logrus.Debugf("%s %s", authReq.Method, authReq.URL.String())
 	tr := tlsclientconfig.NewTransport()
 	// TODO(runcom): insecure for now to contact the external token service
 	tr.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
@@ -497,8 +498,8 @@ func (c *dockerClient) detectProperties(ctx context.Context) error {
 	ping := func(scheme string) error {
 		url := fmt.Sprintf(resolvedPingV2URL, scheme, c.registry)
 		resp, err := c.makeRequestToResolvedURL(ctx, "GET", url, nil, nil, -1, true)
-		logrus.Debugf("Ping %s err %#v", url, err)
 		if err != nil {
+			logrus.Debugf("Ping %s err %#v", url, err)
 			return err
 		}
 		defer resp.Body.Close()

vendor/github.com/containers/image/docker/docker_image_src.go

@@ -310,7 +310,14 @@ func (s *dockerImageSource) getSignaturesFromAPIExtension(ctx context.Context, i
 
 // deleteImage deletes the named image from the registry, if supported.
 func deleteImage(ctx context.Context, sys *types.SystemContext, ref dockerReference) error {
-	c, err := newDockerClientFromRef(sys, ref, true, "push")
+	// docker/distribution does not document what action should be used for deleting images.
+	//
+	// Current docker/distribution requires "pull" for reading the manifest and "delete" for deleting it.
+	// quay.io requires "push" (an explicit "pull" is unnecessary), does not grant any token (fails parsing the request) if "delete" is included.
+	// OpenShift ignores the action string (both the password and the token is an OpenShift API token identifying a user).
+	//
+	// We have to hard-code a single string, luckily both docker/distribution and quay.io support "*" to mean "everything".
+	c, err := newDockerClientFromRef(sys, ref, true, "*")
 	if err != nil {
 		return err
 	}

vendor/github.com/containers/image/manifest/docker_schema2.go

@@ -57,8 +57,9 @@ type Schema2HealthConfig struct {
 	Test []string `json:",omitempty"`
 
 	// Zero means to inherit. Durations are expressed as integer nanoseconds.
-	Interval time.Duration `json:",omitempty"` // Interval is the time to wait between checks.
-	Timeout  time.Duration `json:",omitempty"` // Timeout is the time to wait before considering the check to have hung.
+	StartPeriod time.Duration `json:",omitempty"` // StartPeriod is the time to wait after starting before running the first check.
+	Interval    time.Duration `json:",omitempty"` // Interval is the time to wait between checks.
+	Timeout     time.Duration `json:",omitempty"` // Timeout is the time to wait before considering the check to have hung.
 
 	// Retries is the number of consecutive failures needed to consider a container as unhealthy.
 	// Zero means inherit.

vendor/github.com/containers/image/pkg/tlsclientconfig/tlsclientconfig.go

@@ -34,11 +34,13 @@ func SetupCertificates(dir string, tlsc *tls.Config) error {
 	for _, f := range fs {
 		fullPath := filepath.Join(dir, f.Name())
 		if strings.HasSuffix(f.Name(), ".crt") {
-			systemPool, err := tlsconfig.SystemCertPool()
-			if err != nil {
-				return errors.Wrap(err, "unable to get system cert pool")
+			if tlsc.RootCAs == nil {
+				systemPool, err := tlsconfig.SystemCertPool()
+				if err != nil {
+					return errors.Wrap(err, "unable to get system cert pool")
+				}
+				tlsc.RootCAs = systemPool
 			}
-			tlsc.RootCAs = systemPool
 			logrus.Debugf(" crt: %s", fullPath)
 			data, err := ioutil.ReadFile(fullPath)
 			if err != nil {

vendor/github.com/containers/storage/containers.go

@@ -192,6 +192,9 @@ func (r *containerStore) Load() error {
 }
 
 func (r *containerStore) Save() error {
+	if !r.Locked() {
+		return errors.New("container store is not locked")
+	}
 	rpath := r.containerspath()
 	if err := os.MkdirAll(filepath.Dir(rpath), 0700); err != nil {
 		return err
@@ -560,3 +563,7 @@ func (r *containerStore) IsReadWrite() bool {
 func (r *containerStore) TouchedSince(when time.Time) bool {
 	return r.lockfile.TouchedSince(when)
 }
+
+func (r *containerStore) Locked() bool {
+	return r.lockfile.Locked()
+}

vendor/github.com/containers/storage/images.go

@@ -219,6 +219,9 @@ func (r *imageStore) Save() error {
 	if !r.IsReadWrite() {
 		return errors.Wrapf(ErrStoreIsReadOnly, "not allowed to modify the image store at %q", r.imagespath())
 	}
+	if !r.Locked() {
+		return errors.New("image store is not locked")
+	}
 	rpath := r.imagespath()
 	if err := os.MkdirAll(filepath.Dir(rpath), 0700); err != nil {
 		return err
@@ -701,3 +704,7 @@ func (r *imageStore) IsReadWrite() bool {
 func (r *imageStore) TouchedSince(when time.Time) bool {
 	return r.lockfile.TouchedSince(when)
 }
+
+func (r *imageStore) Locked() bool {
+	return r.lockfile.Locked()
+}

vendor/github.com/containers/storage/layers.go

@@ -371,6 +371,9 @@ func (r *layerStore) Save() error {
 	if !r.IsReadWrite() {
 		return errors.Wrapf(ErrStoreIsReadOnly, "not allowed to modify the layer store at %q", r.layerspath())
 	}
+	if !r.Locked() {
+		return errors.New("layer store is not locked")
+	}
 	rpath := r.layerspath()
 	if err := os.MkdirAll(filepath.Dir(rpath), 0700); err != nil {
 		return err
@@ -1181,3 +1184,7 @@ func (r *layerStore) IsReadWrite() bool {
 func (r *layerStore) TouchedSince(when time.Time) bool {
 	return r.lockfile.TouchedSince(when)
 }
+
+func (r *layerStore) Locked() bool {
+	return r.lockfile.Locked()
+}

vendor/github.com/containers/storage/lockfile.go

@@ -28,6 +28,9 @@ type Locker interface {
 
 	// IsReadWrite() checks if the lock file is read-write
 	IsReadWrite() bool
+
+	// Locked() checks if lock is locked
+	Locked() bool
 }
 
 var (

vendor/github.com/containers/storage/lockfile_unix.go

@@ -25,9 +25,9 @@ func getLockFile(path string, ro bool) (Locker, error) {
 	}
 	unix.CloseOnExec(fd)
 	if ro {
-		return &lockfile{file: path, fd: uintptr(fd), lw: stringid.GenerateRandomID(), locktype: unix.F_RDLCK}, nil
+		return &lockfile{file: path, fd: uintptr(fd), lw: stringid.GenerateRandomID(), locktype: unix.F_RDLCK, locked: false}, nil
 	}
-	return &lockfile{file: path, fd: uintptr(fd), lw: stringid.GenerateRandomID(), locktype: unix.F_WRLCK}, nil
+	return &lockfile{file: path, fd: uintptr(fd), lw: stringid.GenerateRandomID(), locktype: unix.F_WRLCK, locked: false}, nil
 }
 
 type lockfile struct {
@@ -36,6 +36,7 @@ type lockfile struct {
 	fd       uintptr
 	lw       string
 	locktype int16
+	locked   bool
 }
 
 // Lock locks the lock file
@@ -48,6 +49,7 @@ func (l *lockfile) Lock() {
 		Pid:    int32(os.Getpid()),
 	}
 	l.mu.Lock()
+	l.locked = true
 	for unix.FcntlFlock(l.fd, unix.F_SETLKW, &lk) != nil {
 		time.Sleep(10 * time.Millisecond)
 	}
@@ -65,9 +67,15 @@ func (l *lockfile) Unlock() {
 	for unix.FcntlFlock(l.fd, unix.F_SETLKW, &lk) != nil {
 		time.Sleep(10 * time.Millisecond)
 	}
+	l.locked = false
 	l.mu.Unlock()
 }
 
+// Check if lock is locked
+func (l *lockfile) Locked() bool {
+	return l.locked
+}
+
 // Touch updates the lock file with the UID of the user
 func (l *lockfile) Touch() error {
 	l.lw = stringid.GenerateRandomID()

vendor/github.com/containers/storage/lockfile_windows.go

@@ -9,18 +9,29 @@ import (
 )
 
 func getLockFile(path string, ro bool) (Locker, error) {
-	return &lockfile{}, nil
+	return &lockfile{locked: false}, nil
 }
 
 type lockfile struct {
-	mu   sync.Mutex
-	file string
+	mu     sync.Mutex
+	file   string
+	locked bool
 }
 
 func (l *lockfile) Lock() {
+	l.mu.Lock()
+	l.locked = true
 }
+
 func (l *lockfile) Unlock() {
+	l.locked = false
+	l.mu.Unlock()
 }
+
+func (l *lockfile) Locked() bool {
+	return l.locked
+}
+
 func (l *lockfile) Modified() (bool, error) {
 	return false, nil
 }

vendor/github.com/containers/storage/pkg/archive/example_changes.go

@@ -0,0 +1,97 @@
+// +build ignore
+
+// Simple tool to create an archive stream from an old and new directory
+//
+// By default it will stream the comparison of two temporary directories with junk files
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path"
+
+	"github.com/containers/storage/pkg/archive"
+	"github.com/sirupsen/logrus"
+)
+
+var (
+	flDebug  = flag.Bool("D", false, "debugging output")
+	flNewDir = flag.String("newdir", "", "")
+	flOldDir = flag.String("olddir", "", "")
+	log      = logrus.New()
+)
+
+func main() {
+	flag.Usage = func() {
+		fmt.Println("Produce a tar from comparing two directory paths. By default a demo tar is created of around 200 files (including hardlinks)")
+		fmt.Printf("%s [OPTIONS]\n", os.Args[0])
+		flag.PrintDefaults()
+	}
+	flag.Parse()
+	log.Out = os.Stderr
+	if (len(os.Getenv("DEBUG")) > 0) || *flDebug {
+		logrus.SetLevel(logrus.DebugLevel)
+	}
+	var newDir, oldDir string
+
+	if len(*flNewDir) == 0 {
+		var err error
+		newDir, err = ioutil.TempDir("", "storage-test-newDir")
+		if err != nil {
+			log.Fatal(err)
+		}
+		defer os.RemoveAll(newDir)
+		if _, err := prepareUntarSourceDirectory(100, newDir, true); err != nil {
+			log.Fatal(err)
+		}
+	} else {
+		newDir = *flNewDir
+	}
+
+	if len(*flOldDir) == 0 {
+		oldDir, err := ioutil.TempDir("", "storage-test-oldDir")
+		if err != nil {
+			log.Fatal(err)
+		}
+		defer os.RemoveAll(oldDir)
+	} else {
+		oldDir = *flOldDir
+	}
+
+	changes, err := archive.ChangesDirs(newDir, oldDir)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	a, err := archive.ExportChanges(newDir, changes)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer a.Close()
+
+	i, err := io.Copy(os.Stdout, a)
+	if err != nil && err != io.EOF {
+		log.Fatal(err)
+	}
+	fmt.Fprintf(os.Stderr, "wrote archive of %d bytes", i)
+}
+
+func prepareUntarSourceDirectory(numberOfFiles int, targetPath string, makeLinks bool) (int, error) {
+	fileData := []byte("fooo")
+	for n := 0; n < numberOfFiles; n++ {
+		fileName := fmt.Sprintf("file-%d", n)
+		if err := ioutil.WriteFile(path.Join(targetPath, fileName), fileData, 0700); err != nil {
+			return 0, err
+		}
+		if makeLinks {
+			if err := os.Link(path.Join(targetPath, fileName), path.Join(targetPath, fileName+"-link")); err != nil {
+				return 0, err
+			}
+		}
+	}
+	totalSize := numberOfFiles * len(fileData)
+	return totalSize, nil
+}

vendor/github.com/projectatomic/buildah/vendor.conf

@@ -4,8 +4,8 @@ github.com/BurntSushi/toml master
 github.com/containerd/continuity master
 github.com/containernetworking/cni v0.7.0-alpha1
 github.com/seccomp/containers-golang master
-github.com/containers/image 216acb1bcd2c1abef736ee322e17147ee2b7d76c
-github.com/containers/storage 17c7d1fee5603ccf6dd97edc14162fc1510e7e23
+github.com/containers/image 5df44e095ed826fbe2beeaabb329c749d7d6c3b6
+github.com/containers/storage 9fcbb57eb6c732e7b67003bb8ed861f169d33d63
 github.com/docker/distribution 5f6282db7d65e6d72ad7c2cc66310724a57be716
 github.com/docker/docker 86f080cff0914e9694068ed78d503701667c4c00
 github.com/docker/docker-credential-helpers d68f9aeca33f5fd3f08eeae5e9d175edf4e731d1
@@ -46,12 +46,12 @@ github.com/containers/libpod d20f3a51463ce75d139dd830e19a173906b0b0cb
 github.com/sirupsen/logrus master
 github.com/syndtr/gocapability master
 github.com/tchap/go-patricia master
+github.com/ulikunitz/xz v0.5.4
 github.com/urfave/cli 934abfb2f102315b5794e15ebc7949e4ca253920
 github.com/vbatts/tar-split v0.10.2
 github.com/xeipuuv/gojsonpointer master
 github.com/xeipuuv/gojsonreference master
 github.com/xeipuuv/gojsonschema master
-github.com/ulikunitz/xz v0.5.4
 golang.org/x/crypto master
 golang.org/x/net master
 golang.org/x/sys master