Flatten multipart file transfers

License: MIT
Signed-off-by: Jeromy <jeromyj@gmail.com>

commands/cli/parse.go

@@ -44,7 +44,17 @@ func Parse(input []string, stdin *os.File, root *cmds.Command) (cmds.Request, *c
 		}
 	}
 
-	stringArgs, fileArgs, err := parseArgs(stringVals, stdin, cmd.Arguments, recursive, root)
+	// if '--hidden' is provided, enumerate hidden paths
+	hiddenOpt := req.Option("hidden")
+	hidden := false
+	if hiddenOpt != nil {
+		hidden, _, err = hiddenOpt.Bool()
+		if err != nil {
+			return req, nil, nil, u.ErrCast()
+		}
+	}
+
+	stringArgs, fileArgs, err := parseArgs(stringVals, stdin, cmd.Arguments, recursive, hidden, root)
 	if err != nil {
 		return req, cmd, path, err
 	}
@@ -223,7 +233,7 @@ func parseOpts(args []string, root *cmds.Command) (
 	return
 }
 
-func parseArgs(inputs []string, stdin *os.File, argDefs []cmds.Argument, recursive bool, root *cmds.Command) ([]string, []files.File, error) {
+func parseArgs(inputs []string, stdin *os.File, argDefs []cmds.Argument, recursive, hidden bool, root *cmds.Command) ([]string, []files.File, error) {
 	// ignore stdin on Windows
 	if runtime.GOOS == "windows" {
 		stdin = nil
@@ -308,7 +318,7 @@ func parseArgs(inputs []string, stdin *os.File, argDefs []cmds.Argument, recursi
 				// treat stringArg values as file paths
 				fpath := inputs[0]
 				inputs = inputs[1:]
-				file, err := appendFile(fpath, argDef, recursive)
+				file, err := appendFile(fpath, argDef, recursive, hidden)
 				if err != nil {
 					return nil, nil, err
 				}
@@ -389,7 +399,7 @@ func appendStdinAsString(args []string, stdin *os.File) ([]string, *os.File, err
 const notRecursiveFmtStr = "'%s' is a directory, use the '-%s' flag to specify directories"
 const dirNotSupportedFmtStr = "Invalid path '%s', argument '%s' does not support directories"
 
-func appendFile(fpath string, argDef *cmds.Argument, recursive bool) (files.File, error) {
+func appendFile(fpath string, argDef *cmds.Argument, recursive, hidden bool) (files.File, error) {
 	fpath = filepath.ToSlash(filepath.Clean(fpath))
 
 	if fpath == "." {
@@ -414,7 +424,7 @@ func appendFile(fpath string, argDef *cmds.Argument, recursive bool) (files.File
 		}
 	}
 
-	return files.NewSerialFile(path.Base(fpath), fpath, stat)
+	return files.NewSerialFile(path.Base(fpath), fpath, hidden, stat)
 }
 
 // isTerminal returns true if stdin is a Stdin pipe (e.g. `cat file | ipfs`),

commands/files/multipartfile.go

@@ -1,10 +1,10 @@
 package files
 
 import (
+	"io"
 	"io/ioutil"
 	"mime"
 	"mime/multipart"
-	"net/http"
 	"net/url"
 )
 
@@ -12,7 +12,8 @@ const (
 	multipartFormdataType = "multipart/form-data"
 	multipartMixedType    = "multipart/mixed"
 
-	applicationSymlink = "application/symlink"
+	applicationDirectory = "application/x-directory"
+	applicationSymlink   = "application/symlink"
 
 	contentTypeHeader = "Content-Type"
 )
@@ -45,40 +46,33 @@ func NewFileFromPart(part *multipart.Part) (File, error) {
 		}, nil
 	}
 
-	var params map[string]string
 	var err error
-	f.Mediatype, params, err = mime.ParseMediaType(contentType)
+	f.Mediatype, _, err = mime.ParseMediaType(contentType)
 	if err != nil {
 		return nil, err
 	}
 
-	if f.IsDirectory() {
-		boundary, found := params["boundary"]
-		if !found {
-			return nil, http.ErrMissingBoundary
-		}
-
-		f.Reader = multipart.NewReader(part, boundary)
-	}
-
 	return f, nil
 }
 
 func (f *MultipartFile) IsDirectory() bool {
-	return f.Mediatype == multipartFormdataType || f.Mediatype == multipartMixedType
+	return f.Mediatype == multipartFormdataType || f.Mediatype == applicationDirectory
 }
 
 func (f *MultipartFile) NextFile() (File, error) {
 	if !f.IsDirectory() {
 		return nil, ErrNotDirectory
 	}
+	if f.Reader != nil {
+		part, err := f.Reader.NextPart()
+		if err != nil {
+			return nil, err
+		}
 
-	part, err := f.Reader.NextPart()
-	if err != nil {
-		return nil, err
+		return NewFileFromPart(part)
 	}
 
-	return NewFileFromPart(part)
+	return nil, io.EOF
 }
 
 func (f *MultipartFile) FileName() string {

commands/files/serialfile.go

@@ -6,6 +6,7 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"strings"
 	"syscall"
 )
 
@@ -18,9 +19,10 @@ type serialFile struct {
 	files   []os.FileInfo
 	stat    os.FileInfo
 	current *File
+	hidden  bool
 }
 
-func NewSerialFile(name, path string, stat os.FileInfo) (File, error) {
+func NewSerialFile(name, path string, hidden bool, stat os.FileInfo) (File, error) {
 	switch mode := stat.Mode(); {
 	case mode.IsRegular():
 		file, err := os.Open(path)
@@ -35,7 +37,7 @@ func NewSerialFile(name, path string, stat os.FileInfo) (File, error) {
 		if err != nil {
 			return nil, err
 		}
-		return &serialFile{name, path, contents, stat, nil}, nil
+		return &serialFile{name, path, contents, stat, nil, hidden}, nil
 	case mode&os.ModeSymlink != 0:
 		target, err := os.Readlink(path)
 		if err != nil {
@@ -68,6 +70,15 @@ func (f *serialFile) NextFile() (File, error) {
 	stat := f.files[0]
 	f.files = f.files[1:]
 
+	for !f.hidden && strings.HasPrefix(stat.Name(), ".") {
+		if len(f.files) == 0 {
+			return nil, io.EOF
+		}
+
+		stat = f.files[0]
+		f.files = f.files[1:]
+	}
+
 	// open the next file
 	fileName := filepath.ToSlash(filepath.Join(f.name, stat.Name()))
 	filePath := filepath.ToSlash(filepath.Join(f.path, stat.Name()))
@@ -75,7 +86,7 @@ func (f *serialFile) NextFile() (File, error) {
 	// recursively call the constructor on the next file
 	// if it's a regular file, we will open it as a ReaderFile
 	// if it's a directory, files in it will be opened serially
-	sf, err := NewSerialFile(fileName, filePath, stat)
+	sf, err := NewSerialFile(fileName, filePath, f.hidden, stat)
 	if err != nil {
 		return nil, err
 	}
@@ -94,7 +105,7 @@ func (f *serialFile) FullPath() string {
 }
 
 func (f *serialFile) Read(p []byte) (int, error) {
-	return 0, ErrNotReader
+	return 0, io.EOF
 }
 
 func (f *serialFile) Close() error {

commands/files/slicefile.go

@@ -41,7 +41,7 @@ func (f *SliceFile) FullPath() string {
 }
 
 func (f *SliceFile) Read(p []byte) (int, error) {
-	return 0, ErrNotReader
+	return 0, io.EOF
 }
 
 func (f *SliceFile) Close() error {

commands/http/client.go

@@ -13,7 +13,6 @@ import (
 	"strings"
 
 	cmds "github.com/ipfs/go-ipfs/commands"
-	path "github.com/ipfs/go-ipfs/path"
 	config "github.com/ipfs/go-ipfs/repo/config"
 
 	context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
@@ -86,8 +85,8 @@ func (c *client) Send(req cmds.Request) (cmds.Response, error) {
 		reader = fileReader
 	}
 
-	pth := path.Join(req.Path())
-	url := fmt.Sprintf(ApiUrlFormat, c.serverAddress, ApiPath, pth, query)
+	path := strings.Join(req.Path(), "/")
+	url := fmt.Sprintf(ApiUrlFormat, c.serverAddress, ApiPath, path, query)
 
 	httpReq, err := http.NewRequest("POST", url, reader)
 	if err != nil {

commands/http/multifilereader.go

@@ -17,7 +17,7 @@ import (
 type MultiFileReader struct {
 	io.Reader
 
-	files       files.File
+	files       []files.File
 	currentFile io.Reader
 	buf         bytes.Buffer
 	mpWriter    *multipart.Writer
@@ -34,7 +34,7 @@ type MultiFileReader struct {
 // if `form` is false, the Content-Type will be 'multipart/mixed'.
 func NewMultiFileReader(file files.File, form bool) *MultiFileReader {
 	mfr := &MultiFileReader{
-		files: file,
+		files: []files.File{file},
 		form:  form,
 		mutex: &sync.Mutex{},
 	}
@@ -54,34 +54,41 @@ func (mfr *MultiFileReader) Read(buf []byte) (written int, err error) {
 
 	// if the current file isn't set, advance to the next file
 	if mfr.currentFile == nil {
-		file, err := mfr.files.NextFile()
-		if err == io.EOF {
-			mfr.mpWriter.Close()
-			mfr.closed = true
-		} else if err != nil {
-			return 0, err
+		var file files.File
+		for file == nil {
+			if len(mfr.files) == 0 {
+				mfr.mpWriter.Close()
+				mfr.closed = true
+				return mfr.buf.Read(buf)
+			}
+
+			nextfile, err := mfr.files[len(mfr.files)-1].NextFile()
+			if err == io.EOF {
+				mfr.files = mfr.files[:len(mfr.files)-1]
+				continue
+			} else if err != nil {
+				return 0, err
+			}
+
+			file = nextfile
 		}
 
 		// handle starting a new file part
 		if !mfr.closed {
 
 			var contentType string
-			if s, ok := file.(*files.Symlink); ok {
-				mfr.currentFile = s
-
+			if _, ok := file.(*files.Symlink); ok {
 				contentType = "application/symlink"
 			} else if file.IsDirectory() {
-				// if file is a directory, create a multifilereader from it
-				// (using 'multipart/mixed')
-				nmfr := NewMultiFileReader(file, false)
-				mfr.currentFile = nmfr
-				contentType = fmt.Sprintf("multipart/mixed; boundary=%s", nmfr.Boundary())
+				mfr.files = append(mfr.files, file)
+				contentType = "application/x-directory"
 			} else {
 				// otherwise, use the file as a reader to read its contents
-				mfr.currentFile = file
 				contentType = "application/octet-stream"
 			}
 
+			mfr.currentFile = file
+
 			// write the boundary and headers
 			header := make(textproto.MIMEHeader)
 			filename := url.QueryEscape(file.FileName())

core/commands/add.go

@@ -2,7 +2,6 @@ package commands
 
 import (
 	"fmt"
-	"io"
 
 	"github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/cheggaaa/pb"
 	"github.com/ipfs/go-ipfs/core/coreunix"
@@ -147,26 +146,8 @@ remains to be implemented.
 		fileAdder.Pin = dopin
 		fileAdder.Silent = silent
 
-		// addAllFiles loops over a convenience slice file to
-		// add each file individually. e.g. 'ipfs add a b c'
-		addAllFiles := func(sliceFile files.File) error {
-			for {
-				file, err := sliceFile.NextFile()
-				if err != nil && err != io.EOF {
-					return err
-				}
-				if file == nil {
-					return nil // done
-				}
-
-				if err := fileAdder.AddFile(file); err != nil {
-					return err
-				}
-			}
-		}
-
 		addAllAndPin := func(f files.File) error {
-			if err := addAllFiles(f); err != nil {
+			if err := fileAdder.AddFile(f); err != nil {
 				return err
 			}
 

core/coreunix/add.go

@@ -255,7 +255,7 @@ func AddR(n *core.IpfsNode, root string) (key string, err error) {
 		return "", err
 	}
 
-	f, err := files.NewSerialFile(root, root, stat)
+	f, err := files.NewSerialFile(root, root, false, stat)
 	if err != nil {
 		return "", err
 	}
@@ -354,7 +354,7 @@ func (adder *Adder) addFile(file files.File) error {
 
 	switch {
 	case files.IsHidden(file) && !adder.Hidden:
-		log.Debugf("%s is hidden, skipping", file.FileName())
+		log.Infof("%s is hidden, skipping", file.FileName())
 		return &hiddenFileError{file.FileName()}
 	case file.IsDirectory():
 		return adder.addDir(file)

exchange/bitswap/workers.go

@@ -89,7 +89,7 @@ func (bs *Bitswap) provideWorker(px process.Process) {
 		defer cancel()
 
 		if err := bs.network.Provide(ctx, k); err != nil {
-			log.Error(err)
+			log.Warning(err)
 		}
 	}
 

importer/importer.go

@@ -29,7 +29,7 @@ func BuildDagFromFile(fpath string, ds dag.DAGService) (*dag.Node, error) {
 		return nil, fmt.Errorf("`%s` is a directory", fpath)
 	}
 
-	f, err := files.NewSerialFile(fpath, fpath, stat)
+	f, err := files.NewSerialFile(fpath, fpath, false, stat)
 	if err != nil {
 		return nil, err
 	}

mfs/ops.go

@@ -102,7 +102,7 @@ func PutNode(r *Root, path string, nd *dag.Node) error {
 // intermediary directories as needed if 'parents' is set to true
 func Mkdir(r *Root, pth string, parents bool) error {
 	if pth == "" {
-		panic("empty path")
+		return nil
 	}
 	parts := path.SplitList(pth)
 	if parts[0] == "" {