Vendor in latest containers/storage

Signed-off-by: Daniel J Walsh <dwalsh@redhat.com>

Closes: #1061
Approved by: baude

vendor/github.com/containers/storage/README.md

@@ -41,3 +41,6 @@ memory and stored along with the library's own bookkeeping information.
 Additionally, the library can store one or more of what it calls *big data* for
 images and containers.  This is a named chunk of larger data, which is only in
 memory when it is being read from or being written to its own disk file.
+
+**[Contributing](CONTRIBUTING.md)**
+Information about contributing to this project.

vendor/github.com/containers/storage/drivers/overlay/overlay.go

@@ -24,6 +24,7 @@ import (
 	"github.com/containers/storage/pkg/idtools"
 	"github.com/containers/storage/pkg/locker"
 	"github.com/containers/storage/pkg/mount"
+	"github.com/containers/storage/pkg/ostree"
 	"github.com/containers/storage/pkg/parsers"
 	"github.com/containers/storage/pkg/system"
 	units "github.com/docker/go-units"
@@ -84,6 +85,9 @@ type overlayOptions struct {
 	overrideKernelCheck bool
 	imageStores         []string
 	quota               quota.Quota
+	fuseProgram         string
+	ostreeRepo          string
+	skipMountHome       bool
 }
 
 // Driver contains information about the home directory and the list of active mounts that are created using this driver.
@@ -98,6 +102,7 @@ type Driver struct {
 	naiveDiff     graphdriver.DiffDriver
 	supportsDType bool
 	locker        *locker.Locker
+	convert       map[string]bool
 }
 
 var (
@@ -147,15 +152,28 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap
 		return nil, err
 	}
 
-	supportsDType, err := supportsOverlay(home, fsMagic, rootUID, rootGID)
-	if err != nil {
-		os.Remove(filepath.Join(home, linkDir))
-		os.Remove(home)
-		return nil, errors.Wrap(err, "kernel does not support overlay fs")
+	var supportsDType bool
+	if opts.fuseProgram != "" {
+		supportsDType = true
+	} else {
+		supportsDType, err = supportsOverlay(home, fsMagic, rootUID, rootGID)
+		if err != nil {
+			os.Remove(filepath.Join(home, linkDir))
+			os.Remove(home)
+			return nil, errors.Wrap(err, "kernel does not support overlay fs")
+		}
 	}
 
-	if err := mount.MakePrivate(home); err != nil {
-		return nil, err
+	if !opts.skipMountHome {
+		if err := mount.MakePrivate(home); err != nil {
+			return nil, err
+		}
+	}
+
+	if opts.ostreeRepo != "" {
+		if err := ostree.CreateOSTreeRepository(opts.ostreeRepo, rootUID, rootGID); err != nil {
+			return nil, err
+		}
 	}
 
 	d := &Driver{
@@ -167,6 +185,7 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap
 		supportsDType: supportsDType,
 		locker:        locker.New(),
 		options:       *opts,
+		convert:       make(map[string]bool),
 	}
 
 	d.naiveDiff = graphdriver.NewNaiveDiffDriver(d, d)
@@ -227,6 +246,25 @@ func parseOptions(options []string) (*overlayOptions, error) {
 				}
 				o.imageStores = append(o.imageStores, store)
 			}
+		case ".fuse_program", "overlay.fuse_program", "overlay2.fuse_program":
+			logrus.Debugf("overlay: fuse_program=%s", val)
+			_, err := os.Stat(val)
+			if err != nil {
+				return nil, fmt.Errorf("overlay: can't stat FUSE program %s: %v", val, err)
+			}
+			o.fuseProgram = val
+		case "overlay2.ostree_repo", "overlay.ostree_repo", ".ostree_repo":
+			logrus.Debugf("overlay: ostree_repo=%s", val)
+			if !ostree.OstreeSupport() {
+				return nil, fmt.Errorf("overlay: ostree_repo specified but support for ostree is missing")
+			}
+			o.ostreeRepo = val
+		case "overlay2.skip_mount_home", "overlay.skip_mount_home", ".skip_mount_home":
+			logrus.Debugf("overlay: skip_mount_home=%s", val)
+			o.skipMountHome, err = strconv.ParseBool(val)
+			if err != nil {
+				return nil, err
+			}
 		default:
 			return nil, fmt.Errorf("overlay: Unknown option %s", key)
 		}
@@ -236,6 +274,7 @@ func parseOptions(options []string) (*overlayOptions, error) {
 
 func supportsOverlay(home string, homeMagic graphdriver.FsMagic, rootUID, rootGID int) (supportsDType bool, err error) {
 	// We can try to modprobe overlay first
+
 	exec.Command("modprobe", "overlay").Run()
 
 	layerDir, err := ioutil.TempDir(home, "compat")
@@ -380,6 +419,11 @@ func (d *Driver) Create(id, parent string, opts *graphdriver.CreateOpts) (retErr
 			return fmt.Errorf("--storage-opt size is only supported for ReadWrite Layers")
 		}
 	}
+
+	if d.options.ostreeRepo != "" {
+		d.convert[id] = true
+	}
+
 	return d.create(id, parent, opts)
 }
 
@@ -547,6 +591,12 @@ func (d *Driver) getLowerDirs(id string) ([]string, error) {
 func (d *Driver) Remove(id string) error {
 	d.locker.Lock(id)
 	defer d.locker.Unlock(id)
+
+	// Ignore errors, we don't want to fail if the ostree branch doesn't exist,
+	if d.options.ostreeRepo != "" {
+		ostree.DeleteOSTree(d.options.ostreeRepo, id)
+	}
+
 	dir := d.dir(id)
 	lid, err := ioutil.ReadFile(path.Join(dir, "link"))
 	if err == nil {
@@ -663,7 +713,7 @@ func (d *Driver) Get(id, mountLabel string) (_ string, retErr error) {
 	// the page size. The mount syscall fails if the mount data cannot
 	// fit within a page and relative links make the mount data much
 	// smaller at the expense of requiring a fork exec to chroot.
-	if len(mountData) > pageSize {
+	if len(mountData) > pageSize || d.options.fuseProgram != "" {
 		//FIXME: We need to figure out to get this to work with additional stores
 		opts = fmt.Sprintf("lowerdir=%s,upperdir=%s,workdir=%s", strings.Join(relLowers, ":"), path.Join(id, "diff"), path.Join(id, "work"))
 		mountData = label.FormatMountLabel(opts, mountLabel)
@@ -671,8 +721,16 @@ func (d *Driver) Get(id, mountLabel string) (_ string, retErr error) {
 			return "", fmt.Errorf("cannot mount layer, mount label too large %d", len(mountData))
 		}
 
-		mount = func(source string, target string, mType string, flags uintptr, label string) error {
-			return mountFrom(d.home, source, target, mType, flags, label)
+		if d.options.fuseProgram != "" {
+			mount = func(source string, target string, mType string, flags uintptr, label string) error {
+				cmdRootless := exec.Command(d.options.fuseProgram, "-o", label, target)
+				cmdRootless.Dir = d.home
+				return cmdRootless.Run()
+			}
+		} else {
+			mount = func(source string, target string, mType string, flags uintptr, label string) error {
+				return mountFrom(d.home, source, target, mType, flags, label)
+			}
 		}
 		mountTarget = path.Join(id, "merged")
 	}
@@ -764,6 +822,13 @@ func (d *Driver) ApplyDiff(id string, idMappings *idtools.IDMappings, parent str
 		return 0, err
 	}
 
+	_, convert := d.convert[id]
+	if convert {
+		if err := ostree.ConvertToOSTree(d.options.ostreeRepo, applyDir, id); err != nil {
+			return 0, err
+		}
+	}
+
 	return directory.Size(applyDir)
 }
 

vendor/github.com/containers/storage/drivers/vfs/driver.go

@@ -9,6 +9,7 @@ import (
 	"github.com/containers/storage/drivers"
 	"github.com/containers/storage/pkg/chrootarchive"
 	"github.com/containers/storage/pkg/idtools"
+	"github.com/containers/storage/pkg/ostree"
 	"github.com/containers/storage/pkg/system"
 	"github.com/opencontainers/selinux/go-selinux/label"
 )
@@ -42,6 +43,27 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap
 			d.homes = append(d.homes, strings.Split(option[12:], ",")...)
 			continue
 		}
+		if strings.HasPrefix(option, "vfs.ostree_repo=") {
+			if !ostree.OstreeSupport() {
+				return nil, fmt.Errorf("vfs: ostree_repo specified but support for ostree is missing")
+			}
+			d.ostreeRepo = option[16:]
+		}
+		if strings.HasPrefix(option, ".ostree_repo=") {
+			if !ostree.OstreeSupport() {
+				return nil, fmt.Errorf("vfs: ostree_repo specified but support for ostree is missing")
+			}
+			d.ostreeRepo = option[13:]
+		}
+	}
+	if d.ostreeRepo != "" {
+		rootUID, rootGID, err := idtools.GetRootUIDGID(uidMaps, gidMaps)
+		if err != nil {
+			return nil, err
+		}
+		if err := ostree.CreateOSTreeRepository(d.ostreeRepo, rootUID, rootGID); err != nil {
+			return nil, err
+		}
 	}
 	return graphdriver.NewNaiveDiffDriver(d, graphdriver.NewNaiveLayerIDMapUpdater(d)), nil
 }
@@ -53,6 +75,7 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap
 type Driver struct {
 	homes      []string
 	idMappings *idtools.IDMappings
+	ostreeRepo string
 }
 
 func (d *Driver) String() string {
@@ -77,11 +100,15 @@ func (d *Driver) Cleanup() error {
 // CreateReadWrite creates a layer that is writable for use as a container
 // file system.
 func (d *Driver) CreateReadWrite(id, parent string, opts *graphdriver.CreateOpts) error {
-	return d.Create(id, parent, opts)
+	return d.create(id, parent, opts, false)
 }
 
 // Create prepares the filesystem for the VFS driver and copies the directory for the given id under the parent.
 func (d *Driver) Create(id, parent string, opts *graphdriver.CreateOpts) error {
+	return d.create(id, parent, opts, true)
+}
+
+func (d *Driver) create(id, parent string, opts *graphdriver.CreateOpts, ro bool) error {
 	if opts != nil && len(opts.StorageOpt) != 0 {
 		return fmt.Errorf("--storage-opt is not supported for vfs")
 	}
@@ -106,14 +133,23 @@ func (d *Driver) Create(id, parent string, opts *graphdriver.CreateOpts) error {
 	if _, mountLabel, err := label.InitLabels(labelOpts); err == nil {
 		label.SetFileLabel(dir, mountLabel)
 	}
-	if parent == "" {
-		return nil
+	if parent != "" {
+		parentDir, err := d.Get(parent, "")
+		if err != nil {
+			return fmt.Errorf("%s: %s", parent, err)
+		}
+		if err := CopyWithTar(parentDir, dir); err != nil {
+			return err
+		}
 	}
-	parentDir, err := d.Get(parent, "")
-	if err != nil {
-		return fmt.Errorf("%s: %s", parent, err)
+
+	if ro && d.ostreeRepo != "" {
+		if err := ostree.ConvertToOSTree(d.ostreeRepo, dir, id); err != nil {
+			return err
+		}
 	}
-	return CopyWithTar(parentDir, dir)
+	return nil
+
 }
 
 func (d *Driver) dir(id string) string {
@@ -132,6 +168,10 @@ func (d *Driver) dir(id string) string {
 
 // Remove deletes the content from the directory for a given id.
 func (d *Driver) Remove(id string) error {
+	if d.ostreeRepo != "" {
+		// Ignore errors, we don't want to fail if the ostree branch doesn't exist,
+		ostree.DeleteOSTree(d.ostreeRepo, id)
+	}
 	return system.EnsureRemoveAll(d.dir(id))
 }
 

vendor/github.com/containers/storage/pkg/archive/example_changes.go

@@ -0,0 +1,97 @@
+// +build ignore
+
+// Simple tool to create an archive stream from an old and new directory
+//
+// By default it will stream the comparison of two temporary directories with junk files
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path"
+
+	"github.com/containers/storage/pkg/archive"
+	"github.com/sirupsen/logrus"
+)
+
+var (
+	flDebug  = flag.Bool("D", false, "debugging output")
+	flNewDir = flag.String("newdir", "", "")
+	flOldDir = flag.String("olddir", "", "")
+	log      = logrus.New()
+)
+
+func main() {
+	flag.Usage = func() {
+		fmt.Println("Produce a tar from comparing two directory paths. By default a demo tar is created of around 200 files (including hardlinks)")
+		fmt.Printf("%s [OPTIONS]\n", os.Args[0])
+		flag.PrintDefaults()
+	}
+	flag.Parse()
+	log.Out = os.Stderr
+	if (len(os.Getenv("DEBUG")) > 0) || *flDebug {
+		logrus.SetLevel(logrus.DebugLevel)
+	}
+	var newDir, oldDir string
+
+	if len(*flNewDir) == 0 {
+		var err error
+		newDir, err = ioutil.TempDir("", "storage-test-newDir")
+		if err != nil {
+			log.Fatal(err)
+		}
+		defer os.RemoveAll(newDir)
+		if _, err := prepareUntarSourceDirectory(100, newDir, true); err != nil {
+			log.Fatal(err)
+		}
+	} else {
+		newDir = *flNewDir
+	}
+
+	if len(*flOldDir) == 0 {
+		oldDir, err := ioutil.TempDir("", "storage-test-oldDir")
+		if err != nil {
+			log.Fatal(err)
+		}
+		defer os.RemoveAll(oldDir)
+	} else {
+		oldDir = *flOldDir
+	}
+
+	changes, err := archive.ChangesDirs(newDir, oldDir)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	a, err := archive.ExportChanges(newDir, changes)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer a.Close()
+
+	i, err := io.Copy(os.Stdout, a)
+	if err != nil && err != io.EOF {
+		log.Fatal(err)
+	}
+	fmt.Fprintf(os.Stderr, "wrote archive of %d bytes", i)
+}
+
+func prepareUntarSourceDirectory(numberOfFiles int, targetPath string, makeLinks bool) (int, error) {
+	fileData := []byte("fooo")
+	for n := 0; n < numberOfFiles; n++ {
+		fileName := fmt.Sprintf("file-%d", n)
+		if err := ioutil.WriteFile(path.Join(targetPath, fileName), fileData, 0700); err != nil {
+			return 0, err
+		}
+		if makeLinks {
+			if err := os.Link(path.Join(targetPath, fileName), path.Join(targetPath, fileName+"-link")); err != nil {
+				return 0, err
+			}
+		}
+	}
+	totalSize := numberOfFiles * len(fileData)
+	return totalSize, nil
+}

vendor/github.com/containers/storage/pkg/ostree/no_ostree.go

@@ -0,0 +1,19 @@
+// +build !ostree
+
+package ostree
+
+func OstreeSupport() bool {
+	return false
+}
+
+func DeleteOSTree(repoLocation, id string) error {
+	return nil
+}
+
+func CreateOSTreeRepository(repoLocation string, rootUID int, rootGID int) error {
+	return nil
+}
+
+func ConvertToOSTree(repoLocation, root, id string) error {
+	return nil
+}

vendor/github.com/containers/storage/pkg/ostree/ostree.go

@@ -0,0 +1,198 @@
+// +build ostree
+
+package ostree
+
+import (
+	"fmt"
+	"golang.org/x/sys/unix"
+	"os"
+	"path/filepath"
+	"runtime"
+	"syscall"
+	"time"
+	"unsafe"
+
+	"github.com/containers/storage/pkg/idtools"
+	"github.com/containers/storage/pkg/system"
+	glib "github.com/ostreedev/ostree-go/pkg/glibobject"
+	"github.com/ostreedev/ostree-go/pkg/otbuiltin"
+	"github.com/pkg/errors"
+)
+
+// #cgo pkg-config: glib-2.0 gobject-2.0 ostree-1
+// #include <glib.h>
+// #include <glib-object.h>
+// #include <gio/gio.h>
+// #include <stdlib.h>
+// #include <ostree.h>
+// #include <gio/ginputstream.h>
+import "C"
+
+func OstreeSupport() bool {
+	return true
+}
+
+func fixFiles(dir string, usermode bool) (bool, []string, error) {
+	var SkipOstree = errors.New("skip ostree deduplication")
+
+	var whiteouts []string
+
+	err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+		if info.Mode()&(os.ModeNamedPipe|os.ModeSocket|os.ModeDevice) != 0 {
+			if !usermode {
+				stat, ok := info.Sys().(*syscall.Stat_t)
+				if !ok {
+					return errors.New("not syscall.Stat_t")
+				}
+
+				if stat.Rdev == 0 && (stat.Mode&unix.S_IFCHR) != 0 {
+					whiteouts = append(whiteouts, path)
+					return nil
+				}
+			}
+			// Skip the ostree deduplication if we encounter a file type that
+			// ostree does not manage.
+			return SkipOstree
+		}
+		if info.IsDir() {
+			if usermode {
+				if err := os.Chmod(path, info.Mode()|0700); err != nil {
+					return err
+				}
+			}
+		} else if usermode && (info.Mode().IsRegular()) {
+			if err := os.Chmod(path, info.Mode()|0600); err != nil {
+				return err
+			}
+		}
+		return nil
+	})
+	if err == SkipOstree {
+		return true, nil, nil
+	}
+	if err != nil {
+		return false, nil, err
+	}
+	return false, whiteouts, nil
+}
+
+// Create prepares the filesystem for the OSTREE driver and copies the directory for the given id under the parent.
+func ConvertToOSTree(repoLocation, root, id string) error {
+	runtime.LockOSThread()
+	defer runtime.UnlockOSThread()
+	repo, err := otbuiltin.OpenRepo(repoLocation)
+	if err != nil {
+		return errors.Wrap(err, "could not open the OSTree repository")
+	}
+
+	skip, whiteouts, err := fixFiles(root, os.Getuid() != 0)
+	if err != nil {
+		return errors.Wrap(err, "could not prepare the OSTree directory")
+	}
+	if skip {
+		return nil
+	}
+
+	if _, err := repo.PrepareTransaction(); err != nil {
+		return errors.Wrap(err, "could not prepare the OSTree transaction")
+	}
+
+	if skip {
+		return nil
+	}
+
+	commitOpts := otbuiltin.NewCommitOptions()
+	commitOpts.Timestamp = time.Now()
+	commitOpts.LinkCheckoutSpeedup = true
+	commitOpts.Parent = "0000000000000000000000000000000000000000000000000000000000000000"
+	branch := fmt.Sprintf("containers-storage/%s", id)
+
+	for _, w := range whiteouts {
+		if err := os.Remove(w); err != nil {
+			return errors.Wrap(err, "could not delete whiteout file")
+		}
+	}
+
+	if _, err := repo.Commit(root, branch, commitOpts); err != nil {
+		return errors.Wrap(err, "could not commit the layer")
+	}
+
+	if _, err := repo.CommitTransaction(); err != nil {
+		return errors.Wrap(err, "could not complete the OSTree transaction")
+	}
+
+	if err := system.EnsureRemoveAll(root); err != nil {
+		return errors.Wrap(err, "could not delete layer")
+	}
+
+	checkoutOpts := otbuiltin.NewCheckoutOptions()
+	checkoutOpts.RequireHardlinks = true
+	checkoutOpts.Whiteouts = false
+	if err := otbuiltin.Checkout(repoLocation, root, branch, checkoutOpts); err != nil {
+		return errors.Wrap(err, "could not checkout from OSTree")
+	}
+
+	for _, w := range whiteouts {
+		if err := unix.Mknod(w, unix.S_IFCHR, 0); err != nil {
+			return errors.Wrap(err, "could not recreate whiteout file")
+		}
+	}
+	return nil
+}
+
+func CreateOSTreeRepository(repoLocation string, rootUID int, rootGID int) error {
+	runtime.LockOSThread()
+	defer runtime.UnlockOSThread()
+
+	_, err := os.Stat(repoLocation)
+	if err != nil && !os.IsNotExist(err) {
+		return err
+	} else if err != nil {
+		if err := idtools.MkdirAllAs(repoLocation, 0700, rootUID, rootGID); err != nil {
+			return errors.Wrap(err, "could not create OSTree repository directory: %v")
+		}
+
+		if _, err := otbuiltin.Init(repoLocation, otbuiltin.NewInitOptions()); err != nil {
+			return errors.Wrap(err, "could not create OSTree repository")
+		}
+	}
+	return nil
+}
+
+func openRepo(path string) (*C.struct_OstreeRepo, error) {
+	var cerr *C.GError
+	cpath := C.CString(path)
+	defer C.free(unsafe.Pointer(cpath))
+	pathc := C.g_file_new_for_path(cpath)
+	defer C.g_object_unref(C.gpointer(pathc))
+	repo := C.ostree_repo_new(pathc)
+	r := glib.GoBool(glib.GBoolean(C.ostree_repo_open(repo, nil, &cerr)))
+	if !r {
+		C.g_object_unref(C.gpointer(repo))
+		return nil, glib.ConvertGError(glib.ToGError(unsafe.Pointer(cerr)))
+	}
+	return repo, nil
+}
+
+func DeleteOSTree(repoLocation, id string) error {
+	runtime.LockOSThread()
+	defer runtime.UnlockOSThread()
+
+	repo, err := openRepo(repoLocation)
+	if err != nil {
+		return err
+	}
+	defer C.g_object_unref(C.gpointer(repo))
+
+	branch := fmt.Sprintf("containers-storage/%s", id)
+
+	cbranch := C.CString(branch)
+	defer C.free(unsafe.Pointer(cbranch))
+
+	var cerr *C.GError
+	r := glib.GoBool(glib.GBoolean(C.ostree_repo_set_ref_immediate(repo, nil, cbranch, nil, nil, &cerr)))
+	if !r {
+		return glib.ConvertGError(glib.ToGError(unsafe.Pointer(cerr)))
+	}
+	return nil
+}

vendor/github.com/containers/storage/store.go

@@ -2987,6 +2987,11 @@ type OptionsConfig struct {
 	RemapGroup string `toml:"remap-group"`
 	// Thinpool container options to be handed to thinpool drivers
 	Thinpool struct{ ThinpoolOptionsConfig } `toml:"thinpool"`
+	// OSTree repository
+	OstreeRepo string `toml:"ostree_repo"`
+
+	// Do not create a bind mount on the storage home
+	SkipMountHome string `toml:"skip_mount_home"`
 }
 
 // TOML-friendly explicit tables used for conversions.
@@ -3077,6 +3082,12 @@ func init() {
 	if config.Storage.Options.Size != "" {
 		DefaultStoreOptions.GraphDriverOptions = append(DefaultStoreOptions.GraphDriverOptions, fmt.Sprintf("%s.size=%s", config.Storage.Driver, config.Storage.Options.Size))
 	}
+	if config.Storage.Options.OstreeRepo != "" {
+		DefaultStoreOptions.GraphDriverOptions = append(DefaultStoreOptions.GraphDriverOptions, fmt.Sprintf("%s.ostree_repo=%s", config.Storage.Driver, config.Storage.Options.OstreeRepo))
+	}
+	if config.Storage.Options.SkipMountHome != "" {
+		DefaultStoreOptions.GraphDriverOptions = append(DefaultStoreOptions.GraphDriverOptions, fmt.Sprintf("%s.skip_mount_home=%s", config.Storage.Driver, config.Storage.Options.SkipMountHome))
+	}
 	if config.Storage.Options.OverrideKernelCheck != "" {
 		DefaultStoreOptions.GraphDriverOptions = append(DefaultStoreOptions.GraphDriverOptions, fmt.Sprintf("%s.override_kernel_check=%s", config.Storage.Driver, config.Storage.Options.OverrideKernelCheck))
 	}

vendor/github.com/containers/storage/vendor.conf

@@ -20,3 +20,4 @@ github.com/tchap/go-patricia v2.2.6
 github.com/vbatts/tar-split v0.10.2
 golang.org/x/net 7dcfb8076726a3fdd9353b6b8a1f1b6be6811bd6
 golang.org/x/sys 07c182904dbd53199946ba614a412c61d3c548f5
+github.com/ostreedev/ostree-go aeb02c6b6aa2889db3ef62f7855650755befd460