diff --git a/commands/cli/parse.go b/commands/cli/parse.go index 04aa6ed2a..1e1c8b79c 100644 --- a/commands/cli/parse.go +++ b/commands/cli/parse.go @@ -4,6 +4,7 @@ import ( "bytes" "fmt" "os" + "path" "runtime" "strings" @@ -47,7 +48,7 @@ func Parse(input []string, stdin *os.File, root *cmds.Command) (cmds.Request, *c } req.SetArguments(stringArgs) - file := files.NewSliceFile("", fileArgs) + file := files.NewSliceFile("", "", fileArgs) req.SetFiles(file) err = cmd.CheckArguments(req) @@ -341,9 +342,17 @@ func appendStdinAsString(args []string, stdin *os.File) ([]string, *os.File, err } func appendFile(args []files.File, inputs []string, argDef *cmds.Argument, recursive bool) ([]files.File, []string, error) { - path := inputs[0] + fpath := inputs[0] - file, err := os.Open(path) + if fpath == "." { + cwd, err := os.Getwd() + if err != nil { + return nil, nil, err + } + fpath = cwd + } + + file, err := os.Open(fpath) if err != nil { return nil, nil, err } @@ -356,26 +365,25 @@ func appendFile(args []files.File, inputs []string, argDef *cmds.Argument, recur if stat.IsDir() { if !argDef.Recursive { err = fmt.Errorf("Invalid path '%s', argument '%s' does not support directories", - path, argDef.Name) + fpath, argDef.Name) return nil, nil, err } if !recursive { err = fmt.Errorf("'%s' is a directory, use the '-%s' flag to specify directories", - path, cmds.RecShort) + fpath, cmds.RecShort) return nil, nil, err } } - arg, err := files.NewSerialFile(path, file) + arg, err := files.NewSerialFile(path.Base(fpath), fpath, file) if err != nil { return nil, nil, err } - return append(args, arg), inputs[1:], nil } func appendStdinAsFile(args []files.File, stdin *os.File) ([]files.File, *os.File) { - arg := files.NewReaderFile("", stdin, nil) + arg := files.NewReaderFile("", "", stdin, nil) return append(args, arg), nil } diff --git a/commands/files/file.go b/commands/files/file.go index f1196257d..60ced7a71 100644 --- a/commands/files/file.go +++ b/commands/files/file.go @@ -18,9 +18,12 @@ type File interface { // Files implement ReadCloser, but can only be read from or closed if they are not directories io.ReadCloser - // FileName returns a full filename path associated with this file + // FileName returns a filename path associated with this file FileName() string + // FullPath returns the full path in the os associated with this file + FullPath() string + // IsDirectory returns true if the File is a directory (and therefore supports calling `NextFile`) // and false if the File is a normal file (and therefor supports calling `Read` and `Close`) IsDirectory() bool diff --git a/commands/files/file_test.go b/commands/files/file_test.go index 01b7a9d02..395221d4d 100644 --- a/commands/files/file_test.go +++ b/commands/files/file_test.go @@ -11,13 +11,13 @@ import ( func TestSliceFiles(t *testing.T) { name := "testname" files := []File{ - NewReaderFile("file.txt", ioutil.NopCloser(strings.NewReader("Some text!\n")), nil), - NewReaderFile("beep.txt", ioutil.NopCloser(strings.NewReader("beep")), nil), - NewReaderFile("boop.txt", ioutil.NopCloser(strings.NewReader("boop")), nil), + NewReaderFile("file.txt", "file.txt", ioutil.NopCloser(strings.NewReader("Some text!\n")), nil), + NewReaderFile("beep.txt", "beep.txt", ioutil.NopCloser(strings.NewReader("beep")), nil), + NewReaderFile("boop.txt", "boop.txt", ioutil.NopCloser(strings.NewReader("boop")), nil), } buf := make([]byte, 20) - sf := NewSliceFile(name, files) + sf := NewSliceFile(name, name, files) if !sf.IsDirectory() { t.Error("SliceFile should always be a directory") @@ -55,7 +55,7 @@ func TestSliceFiles(t *testing.T) { func TestReaderFiles(t *testing.T) { message := "beep boop" - rf := NewReaderFile("file.txt", ioutil.NopCloser(strings.NewReader(message)), nil) + rf := NewReaderFile("file.txt", "file.txt", ioutil.NopCloser(strings.NewReader(message)), nil) buf := make([]byte, len(message)) if rf.IsDirectory() { diff --git a/commands/files/multipartfile.go b/commands/files/multipartfile.go index ff2aab56b..b68224cbd 100644 --- a/commands/files/multipartfile.go +++ b/commands/files/multipartfile.go @@ -80,6 +80,10 @@ func (f *MultipartFile) FileName() string { return filename } +func (f *MultipartFile) FullPath() string { + return f.FileName() +} + func (f *MultipartFile) Read(p []byte) (int, error) { if f.IsDirectory() { return 0, ErrNotReader diff --git a/commands/files/readerfile.go b/commands/files/readerfile.go index 741042881..7458e82dd 100644 --- a/commands/files/readerfile.go +++ b/commands/files/readerfile.go @@ -10,12 +10,13 @@ import ( // ReaderFiles are never directories, and can be read from and closed. type ReaderFile struct { filename string + fullpath string reader io.ReadCloser stat os.FileInfo } -func NewReaderFile(filename string, reader io.ReadCloser, stat os.FileInfo) *ReaderFile { - return &ReaderFile{filename, reader, stat} +func NewReaderFile(filename, path string, reader io.ReadCloser, stat os.FileInfo) *ReaderFile { + return &ReaderFile{filename, path, reader, stat} } func (f *ReaderFile) IsDirectory() bool { @@ -30,6 +31,10 @@ func (f *ReaderFile) FileName() string { return f.filename } +func (f *ReaderFile) FullPath() string { + return f.fullpath +} + func (f *ReaderFile) Read(p []byte) (int, error) { return f.reader.Read(p) } diff --git a/commands/files/serialfile.go b/commands/files/serialfile.go index 461bde336..4b15e4434 100644 --- a/commands/files/serialfile.go +++ b/commands/files/serialfile.go @@ -18,25 +18,26 @@ func (es sortFIByName) Less(i, j int) bool { return es[i].Name() < es[j].Name() // No more than one file will be opened at a time (directories will advance // to the next file when NextFile() is called). type serialFile struct { + name string path string files []os.FileInfo stat os.FileInfo current *os.File } -func NewSerialFile(path string, file *os.File) (File, error) { +func NewSerialFile(name, path string, file *os.File) (File, error) { stat, err := file.Stat() if err != nil { return nil, err } - return newSerialFile(path, file, stat) + return newSerialFile(name, path, file, stat) } -func newSerialFile(path string, file *os.File, stat os.FileInfo) (File, error) { +func newSerialFile(name, path string, file *os.File, stat os.FileInfo) (File, error) { // for non-directories, return a ReaderFile if !stat.IsDir() { - return &ReaderFile{path, file, stat}, nil + return &ReaderFile{name, path, file, stat}, nil } // for directories, stat all of the contents first, so we know what files to @@ -56,7 +57,7 @@ func newSerialFile(path string, file *os.File, stat os.FileInfo) (File, error) { // make sure contents are sorted so -- repeatably -- we get the same inputs. sort.Sort(sortFIByName(contents)) - return &serialFile{path, contents, stat, nil}, nil + return &serialFile{name, path, contents, stat, nil}, nil } func (f *serialFile) IsDirectory() bool { @@ -81,6 +82,7 @@ func (f *serialFile) NextFile() (File, error) { f.files = f.files[1:] // open the next file + fileName := fp.Join(f.name, stat.Name()) filePath := fp.Join(f.path, stat.Name()) file, err := os.Open(filePath) if err != nil { @@ -91,10 +93,14 @@ func (f *serialFile) NextFile() (File, error) { // recursively call the constructor on the next file // if it's a regular file, we will open it as a ReaderFile // if it's a directory, files in it will be opened serially - return newSerialFile(filePath, file, stat) + return newSerialFile(fileName, filePath, file, stat) } func (f *serialFile) FileName() string { + return f.name +} + +func (f *serialFile) FullPath() string { return f.path } diff --git a/commands/files/slicefile.go b/commands/files/slicefile.go index f6e204812..b705151f1 100644 --- a/commands/files/slicefile.go +++ b/commands/files/slicefile.go @@ -10,12 +10,13 @@ import ( // SliceFiles are always directories, and can't be read from or closed. type SliceFile struct { filename string + path string files []File n int } -func NewSliceFile(filename string, files []File) *SliceFile { - return &SliceFile{filename, files, 0} +func NewSliceFile(filename, path string, files []File) *SliceFile { + return &SliceFile{filename, path, files, 0} } func (f *SliceFile) IsDirectory() bool { @@ -35,6 +36,10 @@ func (f *SliceFile) FileName() string { return f.filename } +func (f *SliceFile) FullPath() string { + return f.path +} + func (f *SliceFile) Read(p []byte) (int, error) { return 0, ErrNotReader } diff --git a/commands/http/multifilereader_test.go b/commands/http/multifilereader_test.go index f1c7aa94d..edd3d6bf2 100644 --- a/commands/http/multifilereader_test.go +++ b/commands/http/multifilereader_test.go @@ -13,14 +13,14 @@ import ( func TestOutput(t *testing.T) { text := "Some text! :)" fileset := []files.File{ - files.NewReaderFile("file.txt", ioutil.NopCloser(strings.NewReader(text)), nil), - files.NewSliceFile("boop", []files.File{ - files.NewReaderFile("boop/a.txt", ioutil.NopCloser(strings.NewReader("bleep")), nil), - files.NewReaderFile("boop/b.txt", ioutil.NopCloser(strings.NewReader("bloop")), nil), + files.NewReaderFile("file.txt", "file.txt", ioutil.NopCloser(strings.NewReader(text)), nil), + files.NewSliceFile("boop", "boop", []files.File{ + files.NewReaderFile("boop/a.txt", "boop/a.txt", ioutil.NopCloser(strings.NewReader("bleep")), nil), + files.NewReaderFile("boop/b.txt", "boop/b.txt", ioutil.NopCloser(strings.NewReader("bloop")), nil), }), - files.NewReaderFile("beep.txt", ioutil.NopCloser(strings.NewReader("beep")), nil), + files.NewReaderFile("beep.txt", "beep.txt", ioutil.NopCloser(strings.NewReader("beep")), nil), } - sf := files.NewSliceFile("", fileset) + sf := files.NewSliceFile("", "", fileset) buf := make([]byte, 20) // testing output by reading it with the go stdlib "mime/multipart" Reader diff --git a/core/commands/add.go b/core/commands/add.go index 614305126..4f87054b6 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -6,6 +6,7 @@ import ( "path" "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/cheggaaa/pb" + cxt "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context" cmds "github.com/ipfs/go-ipfs/commands" files "github.com/ipfs/go-ipfs/commands/files" @@ -13,6 +14,7 @@ import ( importer "github.com/ipfs/go-ipfs/importer" "github.com/ipfs/go-ipfs/importer/chunk" dag "github.com/ipfs/go-ipfs/merkledag" + dagutils "github.com/ipfs/go-ipfs/merkledag/utils" pin "github.com/ipfs/go-ipfs/pin" ft "github.com/ipfs/go-ipfs/unixfs" u "github.com/ipfs/go-ipfs/util" @@ -102,7 +104,7 @@ remains to be implemented. chunker, _, _ := req.Option(chunkerOptionName).String() if hash { - nilnode, err := core.NewNodeBuilder().NilRepo().Build(n.Context()) + nilnode, err := core.NewNodeBuilder().Build(n.Context()) if err != nil { res.SetError(err, cmds.ErrNormal) return @@ -113,22 +115,37 @@ remains to be implemented. outChan := make(chan interface{}, 8) res.SetOutput((<-chan interface{})(outChan)) - // addSingleFile is a function that adds a file given as a param. - addSingleFile := func(file files.File) error { - addParams := adder{ - node: n, - out: outChan, - progress: progress, - hidden: hidden, - trickle: trickle, - chunker: chunker, - } + fileAdder := adder{ + ctx: req.Context(), + node: n, + editor: dagutils.NewDagEditor(n.DAG, newDirNode()), + out: outChan, + chunker: chunker, + progress: progress, + hidden: hidden, + trickle: trickle, + wrap: wrap, + } - rootnd, err := addParams.addFile(file) - if err != nil { - return err - } + // addAllFiles loops over a convenience slice file to + // add each file individually. e.g. 'ipfs add a b c' + addAllFiles := func(sliceFile files.File) error { + for { + file, err := sliceFile.NextFile() + if err != nil && err != io.EOF { + return err + } + if file == nil { + return nil // done + } + if _, err := fileAdder.addFile(file); err != nil { + return err + } + } + } + + pinRoot := func(rootnd *dag.Node) error { rnk, err := rootnd.Key() if err != nil { return err @@ -140,37 +157,22 @@ remains to be implemented. return n.Pinning.Flush() } - // addFilesSeparately loops over a convenience slice file to - // add each file individually. e.g. 'ipfs add a b c' - addFilesSeparately := func(sliceFile files.File) error { - for { - file, err := sliceFile.NextFile() - if err != nil && err != io.EOF { - return err - } - if file == nil { - return nil // done - } - - if err := addSingleFile(file); err != nil { - return err - } + addAllAndPin := func(f files.File) error { + if err := addAllFiles(f); err != nil { + return err } + + rootnd, err := fileAdder.RootNode() + if err != nil { + return err + } + + return pinRoot(rootnd) } go func() { defer close(outChan) - - // really, we're unrapping, if !wrap, because - // req.Files() is already a SliceFile() with all of them, - // so can just use that slice as the wrapper. - var err error - if wrap { - err = addSingleFile(req.Files()) - } else { - err = addFilesSeparately(req.Files()) - } - if err != nil { + if err := addAllAndPin(req.Files()); err != nil { res.SetError(err, cmds.ErrNormal) return } @@ -264,12 +266,17 @@ remains to be implemented. // Internal structure for holding the switches passed to the `add` call type adder struct { + ctx cxt.Context node *core.IpfsNode + editor *dagutils.Editor out chan interface{} progress bool hidden bool trickle bool + wrap bool chunker string + + nextUntitled int } // Perform the actual add & pin locally, outputting results to reader @@ -301,6 +308,40 @@ func add(n *core.IpfsNode, reader io.Reader, useTrickle bool, chunker string) (* return node, nil } +func (params *adder) RootNode() (*dag.Node, error) { + r := params.editor.GetNode() + + // if not wrapping, AND one root file, use that hash as root. + if !params.wrap && len(r.Links) == 1 { + var err error + r, err = r.Links[0].GetNode(params.ctx, params.node.DAG) + // no need to output, as we've already done so. + return r, err + } + + // otherwise need to output, as we have not. + err := outputDagnode(params.out, "", r) + return r, err +} + +func (params *adder) addNode(node *dag.Node, path string) error { + // patch it into the root + key, err := node.Key() + if err != nil { + return err + } + + if path == "" { + path = key.Pretty() + } + + if err := params.editor.InsertNodeAtPath(params.ctx, path, key, newDirNode); err != nil { + return err + } + + return outputDagnode(params.out, path, node) +} + // Add the given file while respecting the params. func (params *adder) addFile(file files.File) (*dag.Node, error) { // Check if file is hidden @@ -326,11 +367,10 @@ func (params *adder) addFile(file files.File) (*dag.Node, error) { return nil, err } + // patch it into the root log.Infof("adding file: %s", file.FileName()) - if err := outputDagnode(params.out, file.FileName(), dagnode); err != nil { - return nil, err - } - return dagnode, nil + err = params.addNode(dagnode, file.FileName()) + return dagnode, err } func (params *adder) addDir(file files.File) (*dag.Node, error) { @@ -364,8 +404,7 @@ func (params *adder) addDir(file files.File) (*dag.Node, error) { } } - err := outputDagnode(params.out, file.FileName(), tree) - if err != nil { + if err := params.addNode(tree, file.FileName()); err != nil { return nil, err } @@ -431,3 +470,8 @@ func (i *progressReader) Read(p []byte) (int, error) { return n, err } + +// TODO: generalize this to more than unix-fs nodes. +func newDirNode() *dag.Node { + return &dag.Node{Data: ft.FolderPBData()} +} diff --git a/core/coreunix/add.go b/core/coreunix/add.go index 17864c9f7..1d3f6cf6b 100644 --- a/core/coreunix/add.go +++ b/core/coreunix/add.go @@ -50,7 +50,7 @@ func AddR(n *core.IpfsNode, root string) (key string, err error) { } defer f.Close() - ff, err := files.NewSerialFile(root, f) + ff, err := files.NewSerialFile(root, root, f) if err != nil { return "", err } @@ -79,8 +79,8 @@ func AddR(n *core.IpfsNode, root string) (key string, err error) { // Returns the path of the added file ("