Merge pull request #8456 from kazimsarikaya/fix-send-tar

podman remote send tar

pkg/bindings/images/build.go

@@ -2,6 +2,7 @@ package images
 
 import (
 	"archive/tar"
+	"compress/gzip"
 	"context"
 	"encoding/json"
 	"io"
@@ -18,6 +19,7 @@ import (
 	"github.com/containers/podman/v2/pkg/auth"
 	"github.com/containers/podman/v2/pkg/bindings"
 	"github.com/containers/podman/v2/pkg/domain/entities"
+	"github.com/containers/storage/pkg/fileutils"
 	"github.com/docker/go-units"
 	"github.com/hashicorp/go-multierror"
 	jsoniter "github.com/json-iterator/go"
@@ -138,12 +140,38 @@ func Build(ctx context.Context, containerFiles []string, options entities.BuildO
 	entries := make([]string, len(containerFiles))
 	copy(entries, containerFiles)
 	entries = append(entries, options.ContextDirectory)
-	tarfile, err := nTar(entries...)
+
+	excludes := options.Excludes
+	if len(excludes) == 0 {
+		excludes, err = parseDockerignore(options.ContextDirectory)
 		if err != nil {
 			return nil, err
 		}
+	}
+
+	tarfile, err := nTar(excludes, entries...)
+	if err != nil {
+		logrus.Errorf("cannot tar container entries %v error: %v", entries, err)
+		return nil, err
+	}
 	defer tarfile.Close()
-	params.Set("dockerfile", filepath.Base(containerFiles[0]))
+
+	containerFile, err := filepath.Abs(entries[0])
+	if err != nil {
+		logrus.Errorf("cannot find absolute path of %v: %v", entries[0], err)
+		return nil, err
+	}
+	contextDir, err := filepath.Abs(entries[1])
+	if err != nil {
+		logrus.Errorf("cannot find absolute path of %v: %v", entries[1], err)
+		return nil, err
+	}
+
+	if strings.HasPrefix(containerFile, contextDir+string(filepath.Separator)) {
+		containerFile = strings.TrimPrefix(containerFile, contextDir+string(filepath.Separator))
+	}
+
+	params.Set("dockerfile", containerFile)
 
 	conn, err := bindings.GetClient(ctx)
 	if err != nil {
@@ -200,35 +228,59 @@ func Build(ctx context.Context, containerFiles []string, options entities.BuildO
 	}
 }
 
-func nTar(sources ...string) (io.ReadCloser, error) {
+func nTar(excludes []string, sources ...string) (io.ReadCloser, error) {
+	pm, err := fileutils.NewPatternMatcher(excludes)
+	if err != nil {
+		return nil, errors.Wrapf(err, "error processing excludes list %v", excludes)
+	}
+
 	if len(sources) == 0 {
 		return nil, errors.New("No source(s) provided for build")
 	}
 
 	pr, pw := io.Pipe()
-	tw := tar.NewWriter(pw)
+	gw := gzip.NewWriter(pw)
+	tw := tar.NewWriter(gw)
 
 	var merr error
 	go func() {
 		defer pw.Close()
+		defer gw.Close()
 		defer tw.Close()
 
 		for _, src := range sources {
-			s := src
+			s, err := filepath.Abs(src)
-			err := filepath.Walk(s, func(path string, info os.FileInfo, err error) error {
+			if err != nil {
+				logrus.Errorf("cannot stat one of source context: %v", err)
+				merr = multierror.Append(merr, err)
+				return
+			}
+
+			err = filepath.Walk(s, func(path string, info os.FileInfo, err error) error {
 				if err != nil {
 					return err
 				}
-				if !info.Mode().IsRegular() || path == s {
+
+				if path == s {
+					return nil // skip root dir
+				}
+
+				name := strings.TrimPrefix(path, s+string(filepath.Separator))
+
+				excluded, err := pm.Matches(filepath.ToSlash(name)) // nolint:staticcheck
+				if err != nil {
+					return errors.Wrapf(err, "error checking if %q is excluded", name)
+				}
+				if excluded {
 					return nil
 				}
 
+				if info.Mode().IsRegular() { // add file item
 					f, lerr := os.Open(path)
 					if lerr != nil {
 						return lerr
 					}
 
-				name := strings.TrimPrefix(path, s+string(filepath.Separator))
 					hdr, lerr := tar.FileInfoHeader(info, name)
 					if lerr != nil {
 						f.Close()
@@ -243,9 +295,49 @@ func nTar(sources ...string) (io.ReadCloser, error) {
 					_, cerr := io.Copy(tw, f)
 					f.Close()
 					return cerr
+				} else if info.Mode().IsDir() { // add folders
+					hdr, lerr := tar.FileInfoHeader(info, name)
+					if lerr != nil {
+						return lerr
+					}
+					hdr.Name = name
+					if lerr := tw.WriteHeader(hdr); lerr != nil {
+						return lerr
+					}
+				} else if info.Mode()&os.ModeSymlink != 0 { // add symlinks as it, not content
+					link, err := os.Readlink(path)
+					if err != nil {
+						return err
+					}
+					hdr, lerr := tar.FileInfoHeader(info, link)
+					if lerr != nil {
+						return lerr
+					}
+					hdr.Name = name
+					if lerr := tw.WriteHeader(hdr); lerr != nil {
+						return lerr
+					}
+				} //skip other than file,folder and symlinks
+				return nil
 			})
 			merr = multierror.Append(merr, err)
 		}
 	}()
 	return pr, merr
 }
+
+func parseDockerignore(root string) ([]string, error) {
+	ignore, err := ioutil.ReadFile(filepath.Join(root, ".dockerignore"))
+	if err != nil && !os.IsNotExist(err) {
+		return nil, errors.Wrapf(err, "error reading .dockerignore: '%s'", root)
+	}
+	rawexcludes := strings.Split(string(ignore), "\n")
+	excludes := make([]string, 0, len(rawexcludes))
+	for _, e := range rawexcludes {
+		if len(e) == 0 || e[0] == '#' {
+			continue
+		}
+		excludes = append(excludes, e)
+	}
+	return excludes, nil
+}

test/e2e/build_test.go

@@ -234,7 +234,7 @@ RUN printenv http_proxy`
 	})
 
 	It("podman build and check identity", func() {
-		session := podmanTest.Podman([]string{"build", "-f", "Containerfile.path", "--no-cache", "-t", "test", "build/basicalpine"})
+		session := podmanTest.Podman([]string{"build", "-f", "build/basicalpine/Containerfile.path", "--no-cache", "-t", "test", "build/basicalpine"})
 		session.WaitWithDefaultTimeout()
 		Expect(session.ExitCode()).To(Equal(0))
 
@@ -244,4 +244,218 @@ RUN printenv http_proxy`
 		data := inspect.OutputToString()
 		Expect(data).To(ContainSubstring(buildah.Version))
 	})
+
+	It("podman remote test container/docker file is not inside context dir", func() {
+		// Given
+		// Switch to temp dir and restore it afterwards
+		cwd, err := os.Getwd()
+		Expect(err).To(BeNil())
+
+		podmanTest.AddImageToRWStore(ALPINE)
+
+		// Write target and fake files
+		targetPath, err := CreateTempDirInTempDir()
+		Expect(err).To(BeNil())
+		targetSubPath := filepath.Join(targetPath, "subdir")
+		err = os.Mkdir(targetSubPath, 0755)
+		Expect(err).To(BeNil())
+		dummyFile := filepath.Join(targetSubPath, "dummy")
+		err = ioutil.WriteFile(dummyFile, []byte("dummy"), 0644)
+		Expect(err).To(BeNil())
+
+		containerfile := `FROM quay.io/libpod/alpine:latest
+ADD . /test
+RUN find /test`
+
+		containerfilePath := filepath.Join(targetPath, "Containerfile")
+		err = ioutil.WriteFile(containerfilePath, []byte(containerfile), 0644)
+		Expect(err).To(BeNil())
+
+		defer func() {
+			Expect(os.Chdir(cwd)).To(BeNil())
+			Expect(os.RemoveAll(targetPath)).To(BeNil())
+		}()
+
+		// make cwd as context root path
+		Expect(os.Chdir(targetPath)).To(BeNil())
+
+		session := podmanTest.Podman([]string{"build", "-t", "test", "-f", "Containerfile", targetSubPath})
+		session.WaitWithDefaultTimeout()
+		Expect(session.ExitCode()).To(Equal(0))
+		ok, _ := session.GrepString("/test/dummy")
+		Expect(ok).To(BeTrue())
+	})
+
+	It("podman remote test container/docker file is not at root of context dir", func() {
+		if IsRemote() {
+			podmanTest.StopRemoteService()
+			podmanTest.StartRemoteService()
+		} else {
+			Skip("Only valid at remote test")
+		}
+		// Given
+		// Switch to temp dir and restore it afterwards
+		cwd, err := os.Getwd()
+		Expect(err).To(BeNil())
+
+		podmanTest.AddImageToRWStore(ALPINE)
+
+		// Write target and fake files
+		targetPath, err := CreateTempDirInTempDir()
+		Expect(err).To(BeNil())
+		targetSubPath := filepath.Join(targetPath, "subdir")
+		err = os.Mkdir(targetSubPath, 0755)
+		Expect(err).To(BeNil())
+
+		containerfile := `FROM quay.io/libpod/alpine:latest`
+
+		containerfilePath := filepath.Join(targetSubPath, "Containerfile")
+		err = ioutil.WriteFile(containerfilePath, []byte(containerfile), 0644)
+		Expect(err).To(BeNil())
+
+		defer func() {
+			Expect(os.Chdir(cwd)).To(BeNil())
+			Expect(os.RemoveAll(targetPath)).To(BeNil())
+		}()
+
+		// make cwd as context root path
+		Expect(os.Chdir(targetPath)).To(BeNil())
+
+		session := podmanTest.Podman([]string{"build", "-t", "test", "-f", "subdir/Containerfile", "."})
+		session.WaitWithDefaultTimeout()
+		Expect(session.ExitCode()).To(Equal(0))
+	})
+
+	It("podman remote test .dockerignore", func() {
+		if IsRemote() {
+			podmanTest.StopRemoteService()
+			podmanTest.StartRemoteService()
+		} else {
+			Skip("Only valid at remote test")
+		}
+		// Given
+		// Switch to temp dir and restore it afterwards
+		cwd, err := os.Getwd()
+		Expect(err).To(BeNil())
+
+		podmanTest.AddImageToRWStore(ALPINE)
+
+		// Write target and fake files
+		targetPath, err := CreateTempDirInTempDir()
+		Expect(err).To(BeNil())
+
+		containerfile := `FROM quay.io/libpod/alpine:latest
+ADD . /testfilter/
+RUN find /testfilter/`
+
+		containerfilePath := filepath.Join(targetPath, "Containerfile")
+		err = ioutil.WriteFile(containerfilePath, []byte(containerfile), 0644)
+		Expect(err).To(BeNil())
+
+		targetSubPath := filepath.Join(targetPath, "subdir")
+		err = os.Mkdir(targetSubPath, 0755)
+		Expect(err).To(BeNil())
+
+		dummyFile1 := filepath.Join(targetPath, "dummy1")
+		err = ioutil.WriteFile(dummyFile1, []byte("dummy1"), 0644)
+		Expect(err).To(BeNil())
+
+		dummyFile2 := filepath.Join(targetPath, "dummy2")
+		err = ioutil.WriteFile(dummyFile2, []byte("dummy2"), 0644)
+		Expect(err).To(BeNil())
+
+		dummyFile3 := filepath.Join(targetSubPath, "dummy3")
+		err = ioutil.WriteFile(dummyFile3, []byte("dummy3"), 0644)
+		Expect(err).To(BeNil())
+
+		defer func() {
+			Expect(os.Chdir(cwd)).To(BeNil())
+			Expect(os.RemoveAll(targetPath)).To(BeNil())
+		}()
+
+		// make cwd as context root path
+		Expect(os.Chdir(targetPath)).To(BeNil())
+
+		dockerignoreContent := `dummy1
+subdir**`
+		dockerignoreFile := filepath.Join(targetPath, ".dockerignore")
+
+		// test .dockerignore
+		By("Test .dockererignore")
+		err = ioutil.WriteFile(dockerignoreFile, []byte(dockerignoreContent), 0644)
+		Expect(err).To(BeNil())
+
+		session := podmanTest.Podman([]string{"build", "-t", "test", "."})
+		session.WaitWithDefaultTimeout()
+		Expect(session.ExitCode()).To(Equal(0))
+		ok, _ := session.GrepString("/testfilter/dummy1")
+		Expect(ok).NotTo(BeTrue())
+		ok, _ = session.GrepString("/testfilter/dummy2")
+		Expect(ok).To(BeTrue())
+		ok, _ = session.GrepString("/testfilter/subdir")
+		Expect(ok).NotTo(BeTrue()) //.dockerignore filters both subdir and inside subdir
+		ok, _ = session.GrepString("/testfilter/subdir/dummy3")
+		Expect(ok).NotTo(BeTrue())
+	})
+
+	It("podman remote test context dir contains empty dirs and symlinks", func() {
+		if IsRemote() {
+			podmanTest.StopRemoteService()
+			podmanTest.StartRemoteService()
+		} else {
+			Skip("Only valid at remote test")
+		}
+		// Given
+		// Switch to temp dir and restore it afterwards
+		cwd, err := os.Getwd()
+		Expect(err).To(BeNil())
+
+		podmanTest.AddImageToRWStore(ALPINE)
+
+		// Write target and fake files
+		targetPath, err := CreateTempDirInTempDir()
+		Expect(err).To(BeNil())
+		targetSubPath := filepath.Join(targetPath, "subdir")
+		err = os.Mkdir(targetSubPath, 0755)
+		Expect(err).To(BeNil())
+		dummyFile := filepath.Join(targetSubPath, "dummy")
+		err = ioutil.WriteFile(dummyFile, []byte("dummy"), 0644)
+		Expect(err).To(BeNil())
+
+		emptyDir := filepath.Join(targetSubPath, "emptyDir")
+		err = os.Mkdir(emptyDir, 0755)
+		Expect(err).To(BeNil())
+		Expect(os.Chdir(targetSubPath)).To(BeNil())
+		Expect(os.Symlink("dummy", "dummy-symlink")).To(BeNil())
+
+		containerfile := `FROM quay.io/libpod/alpine:latest
+ADD . /test
+RUN find /test
+RUN [[ -L /test/dummy-symlink ]] && echo SYMLNKOK || echo SYMLNKERR`
+
+		containerfilePath := filepath.Join(targetSubPath, "Containerfile")
+		err = ioutil.WriteFile(containerfilePath, []byte(containerfile), 0644)
+		Expect(err).To(BeNil())
+
+		defer func() {
+			Expect(os.Chdir(cwd)).To(BeNil())
+			Expect(os.RemoveAll(targetPath)).To(BeNil())
+		}()
+
+		// make cwd as context root path
+		Expect(os.Chdir(targetPath)).To(BeNil())
+
+		session := podmanTest.Podman([]string{"build", "-t", "test", targetSubPath})
+		session.WaitWithDefaultTimeout()
+		Expect(session.ExitCode()).To(Equal(0))
+		ok, _ := session.GrepString("/test/dummy")
+		Expect(ok).To(BeTrue())
+		ok, _ = session.GrepString("/test/emptyDir")
+		Expect(ok).To(BeTrue())
+		ok, _ = session.GrepString("/test/dummy-symlink")
+		Expect(ok).To(BeTrue())
+		ok, _ = session.GrepString("SYMLNKOK")
+		Expect(ok).To(BeTrue())
+	})
+
 })