Merge pull request #3443 from adrianreber/rootfs-changes-migration

Include changes to the container's root file-system in the checkpoint archive

cmd/podman/checkpoint.go

@@ -46,6 +46,7 @@ func init() {
 	flags.BoolVarP(&checkpointCommand.All, "all", "a", false, "Checkpoint all running containers")
 	flags.BoolVarP(&checkpointCommand.Latest, "latest", "l", false, "Act on the latest container podman is aware of")
 	flags.StringVarP(&checkpointCommand.Export, "export", "e", "", "Export the checkpoint image to a tar.gz")
+	flags.BoolVar(&checkpointCommand.IgnoreRootfs, "ignore-rootfs", false, "Do not include root file-system changes when exporting")
 	markFlagHiddenForRemoteClient("latest", flags)
 }
 

cmd/podman/cliconfig/config.go

@@ -92,6 +92,7 @@ type CheckpointValues struct {
 	All            bool
 	Latest         bool
 	Export         string
+	IgnoreRootfs   bool
 }
 
 type CommitValues struct {
@@ -433,6 +434,7 @@ type RestoreValues struct {
 	TcpEstablished bool
 	Import         string
 	Name           string
+	IgnoreRootfs   bool
 }
 
 type RmValues struct {

cmd/podman/restore.go

@@ -45,6 +45,7 @@ func init() {
 	flags.BoolVar(&restoreCommand.TcpEstablished, "tcp-established", false, "Restore a container with established TCP connections")
 	flags.StringVarP(&restoreCommand.Import, "import", "i", "", "Restore from exported checkpoint archive (tar.gz)")
 	flags.StringVarP(&restoreCommand.Name, "name", "n", "", "Specify new name for container restored from exported checkpoint (only works with --import)")
+	flags.BoolVar(&restoreCommand.IgnoreRootfs, "ignore-rootfs", false, "Do not apply root file-system changes when importing from exported checkpoint")
 
 	markFlagHiddenForRemoteClient("latest", flags)
 }
@@ -60,8 +61,12 @@ func restoreCmd(c *cliconfig.RestoreValues, cmd *cobra.Command) error {
 	}
 	defer runtime.DeferredShutdown(false)
 
+	if c.Import == "" && c.IgnoreRootfs {
+		return errors.Errorf("--ignore-rootfs can only be used with --import")
+	}
+
 	if c.Import == "" && c.Name != "" {
-		return errors.Errorf("--name can only used with --import")
+		return errors.Errorf("--name can only be used with --import")
 	}
 
 	if c.Name != "" && c.TcpEstablished {

completions/bash/podman

@@ -758,6 +758,7 @@ _podman_container_checkpoint() {
      -R
      --leave-running
      --tcp-established
+     --ignore-rootfs
      "
      case "$prev" in
         -e|--export)
@@ -870,6 +871,7 @@ _podman_container_restore() {
 	  -l
 	  --latest
 	  --tcp-established
+	  --ignore-rootfs
      "
      case "$prev" in
         -i|--import)

docs/podman-container-checkpoint.1.md

@@ -42,7 +42,15 @@ connections.
 
 Export the checkpoint to a tar.gz file. The exported checkpoint can be used
 to import the container on another system and thus enabling container live
-migration.
+migration. This checkpoint archive also includes all changes to the container's
+root file-system, if not explicitly disabled using **--ignore-rootfs**
+
+**--ignore-rootfs**
+
+This only works in combination with **--export, -e**. If a checkpoint is
+exported to a tar.gz file it is possible with the help of **--ignore-rootfs**
+to explicitly disable including changes to the root file-system into
+the checkpoint archive file.
 
 ## EXAMPLE
 

docs/podman-container-restore.1.md

@@ -60,6 +60,13 @@ address to the container it was using before checkpointing as each IP address ca
 be used once and the restored container will have another IP address. This also means
 that **--name, -n** cannot be used in combination with **--tcp-established**.
 
+**--ignore-rootfs**
+
+This is only available in combination with **--import, -i**. If a container is restored
+from a checkpoint tar.gz file it is possible that it also contains all root file-system
+changes. With **--ignore-rootfs** it is possible to explicitly disable applying these
+root file-system changes to the restored container.
+
 ## EXAMPLE
 
 podman container restore mywebserver

libpod/container_api.go

@@ -801,15 +801,16 @@ type ContainerCheckpointOptions struct {
 	// TCPEstablished tells the API to checkpoint a container
 	// even if it contains established TCP connections
 	TCPEstablished bool
-	// Export tells the API to write the checkpoint image to
-	// the filename set in TargetFile
-	// Import tells the API to read the checkpoint image from
-	// the filename set in TargetFile
+	// TargetFile tells the API to read (or write) the checkpoint image
+	// from (or to) the filename set in TargetFile
 	TargetFile string
 	// Name tells the API that during restore from an exported
 	// checkpoint archive a new name should be used for the
 	// restored container
 	Name string
+	// IgnoreRootfs tells the API to not export changes to
+	// the container's root file-system (or to not import)
+	IgnoreRootfs bool
 }
 
 // Checkpoint checkpoints a container

libpod/container_internal_linux.go

@@ -510,21 +510,44 @@ func (c *Container) addNamespaceContainer(g *generate.Generator, ns LinuxNS, ctr
 	return nil
 }
 
-func (c *Container) exportCheckpoint(dest string) (err error) {
+func (c *Container) exportCheckpoint(dest string, ignoreRootfs bool) (err error) {
 	if (len(c.config.NamedVolumes) > 0) || (len(c.Dependencies()) > 0) {
 		return errors.Errorf("Cannot export checkpoints of containers with named volumes or dependencies")
 	}
 	logrus.Debugf("Exporting checkpoint image of container %q to %q", c.ID(), dest)
-	input, err := archive.TarWithOptions(c.bundlePath(), &archive.TarOptions{
-		Compression:      archive.Gzip,
-		IncludeSourceDir: true,
-		IncludeFiles: []string{
+
+	includeFiles := []string{
 		"checkpoint",
 		"artifacts",
 		"ctr.log",
 		"config.dump",
 		"spec.dump",
-			"network.status"},
+		"network.status"}
+
+	// Get root file-system changes included in the checkpoint archive
+	rootfsDiffPath := filepath.Join(c.bundlePath(), "rootfs-diff.tar")
+	if !ignoreRootfs {
+		rootfsDiffFile, err := os.Create(rootfsDiffPath)
+		if err != nil {
+			return errors.Wrapf(err, "error creating root file-system diff file %q", rootfsDiffPath)
+		}
+		tarStream, err := c.runtime.GetDiffTarStream("", c.ID())
+		if err != nil {
+			return errors.Wrapf(err, "error exporting root file-system diff to %q", rootfsDiffPath)
+		}
+		_, err = io.Copy(rootfsDiffFile, tarStream)
+		if err != nil {
+			return errors.Wrapf(err, "error exporting root file-system diff to %q", rootfsDiffPath)
+		}
+		tarStream.Close()
+		rootfsDiffFile.Close()
+		includeFiles = append(includeFiles, "rootfs-diff.tar")
+	}
+
+	input, err := archive.TarWithOptions(c.bundlePath(), &archive.TarOptions{
+		Compression:      archive.Gzip,
+		IncludeSourceDir: true,
+		IncludeFiles:     includeFiles,
 	})
 
 	if err != nil {
@@ -546,6 +569,8 @@ func (c *Container) exportCheckpoint(dest string) (err error) {
 		return err
 	}
 
+	os.Remove(rootfsDiffPath)
+
 	return nil
 }
 
@@ -605,7 +630,7 @@ func (c *Container) checkpoint(ctx context.Context, options ContainerCheckpointO
 	}
 
 	if options.TargetFile != "" {
-		if err = c.exportCheckpoint(options.TargetFile); err != nil {
+		if err = c.exportCheckpoint(options.TargetFile, options.IgnoreRootfs); err != nil {
 			return err
 		}
 	}
@@ -792,6 +817,23 @@ func (c *Container) restore(ctx context.Context, options ContainerCheckpointOpti
 	if err := c.saveSpec(g.Spec()); err != nil {
 		return err
 	}
+
+	// Before actually restarting the container, apply the root file-system changes
+	if !options.IgnoreRootfs {
+		rootfsDiffPath := filepath.Join(c.bundlePath(), "rootfs-diff.tar")
+		if _, err := os.Stat(rootfsDiffPath); err == nil {
+			// Only do this if a rootfs-diff.tar actually exists
+			rootfsDiffFile, err := os.Open(rootfsDiffPath)
+			if err != nil {
+				return errors.Wrapf(err, "Failed to open root file-system diff file %s", rootfsDiffPath)
+			}
+			if err := c.runtime.ApplyDiffTarStream(c.ID(), rootfsDiffFile); err != nil {
+				return errors.Wrapf(err, "Failed to apply root file-system diff file %s", rootfsDiffPath)
+			}
+			rootfsDiffFile.Close()
+		}
+	}
+
 	if err := c.ociRuntime.createContainer(c, c.config.CgroupParent, &options); err != nil {
 		return err
 	}
@@ -809,7 +851,7 @@ func (c *Container) restore(ctx context.Context, options ContainerCheckpointOpti
 		if err != nil {
 			logrus.Debugf("Non-fatal: removal of checkpoint directory (%s) failed: %v", c.CheckpointPath(), err)
 		}
-		cleanup := [...]string{"restore.log", "dump.log", "stats-dump", "stats-restore", "network.status"}
+		cleanup := [...]string{"restore.log", "dump.log", "stats-dump", "stats-restore", "network.status", "rootfs-diff.tar"}
 		for _, del := range cleanup {
 			file := filepath.Join(c.bundlePath(), del)
 			err = os.Remove(file)

libpod/diff.go

@@ -1,6 +1,9 @@
 package libpod
 
 import (
+	"archive/tar"
+	"io"
+
 	"github.com/containers/libpod/libpod/layers"
 	"github.com/containers/storage/pkg/archive"
 	"github.com/pkg/errors"
@@ -44,6 +47,59 @@ func (r *Runtime) GetDiff(from, to string) ([]archive.Change, error) {
 	return rchanges, err
 }
 
+// skipFileInTarAchive is an archive.TarModifierFunc function
+// which tells archive.ReplaceFileTarWrapper to skip files
+// from the tarstream
+func skipFileInTarAchive(path string, header *tar.Header, content io.Reader) (*tar.Header, []byte, error) {
+	return nil, nil, nil
+}
+
+// GetDiffTarStream returns the differences between the two images, layers, or containers.
+// It is the same functionality as GetDiff() except that it returns a tarstream
+func (r *Runtime) GetDiffTarStream(from, to string) (io.ReadCloser, error) {
+	toLayer, err := r.getLayerID(to)
+	if err != nil {
+		return nil, err
+	}
+	fromLayer := ""
+	if from != "" {
+		fromLayer, err = r.getLayerID(from)
+		if err != nil {
+			return nil, err
+		}
+	}
+	rc, err := r.store.Diff(fromLayer, toLayer, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	// Skip files in the tar archive which are listed
+	// in containerMounts map. Just as in the GetDiff()
+	// function from above
+	filterMap := make(map[string]archive.TarModifierFunc)
+	for key := range containerMounts {
+		filterMap[key[1:]] = skipFileInTarAchive
+		// In the tarstream directories always include a trailing '/'.
+		// For simplicity this duplicates every entry from
+		// containerMounts with a trailing '/', as containerMounts
+		// does not use trailing '/' for directories.
+		filterMap[key[1:]+"/"] = skipFileInTarAchive
+	}
+
+	filteredTarStream := archive.ReplaceFileTarWrapper(rc, filterMap)
+	return filteredTarStream, nil
+}
+
+// ApplyDiffTarStream applies the changes stored in 'diff' to the layer 'to'
+func (r *Runtime) ApplyDiffTarStream(to string, diff io.Reader) error {
+	toLayer, err := r.getLayerID(to)
+	if err != nil {
+		return err
+	}
+	_, err = r.store.ApplyDiff(toLayer, diff)
+	return err
+}
+
 // GetLayerID gets a full layer id given a full or partial id
 // If the id matches a container or image, the id of the top layer is returned
 // If the id matches a layer, the top layer id is returned

pkg/adapter/checkpoint_restore.go

@@ -58,6 +58,7 @@ func crImportCheckpoint(ctx context.Context, runtime *libpod.Runtime, input stri
 			"checkpoint",
 			"artifacts",
 			"ctr.log",
+			"rootfs-diff.tar",
 			"network.status",
 		},
 	}

pkg/adapter/containers.go

@@ -524,6 +524,10 @@ func (r *LocalRuntime) Checkpoint(c *cliconfig.CheckpointValues) error {
 		KeepRunning:    c.LeaveRunning,
 		TCPEstablished: c.TcpEstablished,
 		TargetFile:     c.Export,
+		IgnoreRootfs:   c.IgnoreRootfs,
+	}
+	if c.Export == "" && c.IgnoreRootfs {
+		return errors.Errorf("--ignore-rootfs can only be used with --export")
 	}
 	if c.All {
 		containers, err = r.Runtime.GetRunningContainers()
@@ -560,6 +564,7 @@ func (r *LocalRuntime) Restore(ctx context.Context, c *cliconfig.RestoreValues)
 		TCPEstablished: c.TcpEstablished,
 		TargetFile:     c.Import,
 		Name:           c.Name,
+		IgnoreRootfs:   c.IgnoreRootfs,
 	}
 
 	filterFuncs = append(filterFuncs, func(c *libpod.Container) bool {

pkg/adapter/containers_remote.go

@@ -669,6 +669,9 @@ func (r *LocalRuntime) Checkpoint(c *cliconfig.CheckpointValues) error {
 	if c.Export != "" {
 		return errors.New("the remote client does not support exporting checkpoints")
 	}
+	if c.IgnoreRootfs {
+		return errors.New("the remote client does not support --ignore-rootfs")
+	}
 
 	var lastError error
 	ids, err := iopodman.GetContainersByContext().Call(r.Conn, c.All, c.Latest, c.InputArgs)
@@ -709,6 +712,9 @@ func (r *LocalRuntime) Restore(ctx context.Context, c *cliconfig.RestoreValues)
 	if c.Import != "" {
 		return errors.New("the remote client does not support importing checkpoints")
 	}
+	if c.IgnoreRootfs {
+		return errors.New("the remote client does not support --ignore-rootfs")
+	}
 
 	var lastError error
 	ids, err := iopodman.GetContainersByContext().Call(r.Conn, c.All, c.Latest, c.InputArgs)

test/e2e/checkpoint_test.go

@@ -416,6 +416,130 @@ var _ = Describe("Podman checkpoint", func() {
 		os.Remove(fileName)
 	})
 
+	It("podman checkpoint and restore container with root file-system changes", func() {
+		// Start the container
+		localRunString := getRunString([]string{"--rm", ALPINE, "top"})
+		session := podmanTest.Podman(localRunString)
+		session.WaitWithDefaultTimeout()
+		Expect(session.ExitCode()).To(Equal(0))
+		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(1))
+		cid := session.OutputToString()
+		fileName := "/tmp/checkpoint-" + cid + ".tar.gz"
+
+		// Change the container's root file-system
+		result := podmanTest.Podman([]string{"exec", "-l", "/bin/sh", "-c", "echo test" + cid + "test > /test.output"})
+		result.WaitWithDefaultTimeout()
+		Expect(result.ExitCode()).To(Equal(0))
+
+		// Checkpoint the container
+		result = podmanTest.Podman([]string{"container", "checkpoint", "-l", "-e", fileName})
+		result.WaitWithDefaultTimeout()
+
+		Expect(result.ExitCode()).To(Equal(0))
+		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(0))
+		Expect(podmanTest.NumberOfContainers()).To(Equal(0))
+
+		// Restore the container
+		result = podmanTest.Podman([]string{"container", "restore", "-i", fileName})
+		result.WaitWithDefaultTimeout()
+
+		Expect(result.ExitCode()).To(Equal(0))
+		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(1))
+		Expect(podmanTest.NumberOfContainers()).To(Equal(1))
+		Expect(podmanTest.GetContainerStatus()).To(ContainSubstring("Up"))
+
+		// Verify the changes to the container's root file-system
+		result = podmanTest.Podman([]string{"exec", "-l", "cat", "/test.output"})
+		result.WaitWithDefaultTimeout()
+		Expect(result.ExitCode()).To(Equal(0))
+		Expect(result.OutputToString()).To(ContainSubstring("test" + cid + "test"))
+
+		// Remove exported checkpoint
+		os.Remove(fileName)
+	})
+	It("podman checkpoint and restore container with root file-system changes using --ignore-rootfs during restore", func() {
+		// Start the container
+		localRunString := getRunString([]string{"--rm", ALPINE, "top"})
+		session := podmanTest.Podman(localRunString)
+		session.WaitWithDefaultTimeout()
+		Expect(session.ExitCode()).To(Equal(0))
+		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(1))
+		cid := session.OutputToString()
+		fileName := "/tmp/checkpoint-" + cid + ".tar.gz"
+
+		// Change the container's root file-system
+		result := podmanTest.Podman([]string{"exec", "-l", "/bin/sh", "-c", "echo test" + cid + "test > /test.output"})
+		result.WaitWithDefaultTimeout()
+		Expect(result.ExitCode()).To(Equal(0))
+
+		// Checkpoint the container
+		result = podmanTest.Podman([]string{"container", "checkpoint", "-l", "-e", fileName})
+		result.WaitWithDefaultTimeout()
+
+		Expect(result.ExitCode()).To(Equal(0))
+		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(0))
+		Expect(podmanTest.NumberOfContainers()).To(Equal(0))
+
+		// Restore the container
+		result = podmanTest.Podman([]string{"container", "restore", "--ignore-rootfs", "-i", fileName})
+		result.WaitWithDefaultTimeout()
+
+		Expect(result.ExitCode()).To(Equal(0))
+		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(1))
+		Expect(podmanTest.NumberOfContainers()).To(Equal(1))
+		Expect(podmanTest.GetContainerStatus()).To(ContainSubstring("Up"))
+
+		// Verify the changes to the container's root file-system
+		result = podmanTest.Podman([]string{"exec", "-l", "cat", "/test.output"})
+		result.WaitWithDefaultTimeout()
+		Expect(result.ExitCode()).To(Equal(1))
+		Expect(result.ErrorToString()).To(ContainSubstring("cat: can't open '/test.output': No such file or directory"))
+
+		// Remove exported checkpoint
+		os.Remove(fileName)
+	})
+	It("podman checkpoint and restore container with root file-system changes using --ignore-rootfs during checkpoint", func() {
+		// Start the container
+		localRunString := getRunString([]string{"--rm", ALPINE, "top"})
+		session := podmanTest.Podman(localRunString)
+		session.WaitWithDefaultTimeout()
+		Expect(session.ExitCode()).To(Equal(0))
+		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(1))
+		cid := session.OutputToString()
+		fileName := "/tmp/checkpoint-" + cid + ".tar.gz"
+
+		// Change the container's root file-system
+		result := podmanTest.Podman([]string{"exec", "-l", "/bin/sh", "-c", "echo test" + cid + "test > /test.output"})
+		result.WaitWithDefaultTimeout()
+		Expect(result.ExitCode()).To(Equal(0))
+
+		// Checkpoint the container
+		result = podmanTest.Podman([]string{"container", "checkpoint", "--ignore-rootfs", "-l", "-e", fileName})
+		result.WaitWithDefaultTimeout()
+
+		Expect(result.ExitCode()).To(Equal(0))
+		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(0))
+		Expect(podmanTest.NumberOfContainers()).To(Equal(0))
+
+		// Restore the container
+		result = podmanTest.Podman([]string{"container", "restore", "-i", fileName})
+		result.WaitWithDefaultTimeout()
+
+		Expect(result.ExitCode()).To(Equal(0))
+		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(1))
+		Expect(podmanTest.NumberOfContainers()).To(Equal(1))
+		Expect(podmanTest.GetContainerStatus()).To(ContainSubstring("Up"))
+
+		// Verify the changes to the container's root file-system
+		result = podmanTest.Podman([]string{"exec", "-l", "cat", "/test.output"})
+		result.WaitWithDefaultTimeout()
+		Expect(result.ExitCode()).To(Equal(1))
+		Expect(result.ErrorToString()).To(ContainSubstring("cat: can't open '/test.output': No such file or directory"))
+
+		// Remove exported checkpoint
+		os.Remove(fileName)
+	})
+
 	It("podman checkpoint and run exec in restored container", func() {
 		// Start the container
 		localRunString := getRunString([]string{"--rm", ALPINE, "top"})