Merge pull request #1506 from ipfs/feat/patch-create

allow patch to optionally create intermediate dirs

core/commands/object.go

@@ -18,6 +18,7 @@ import (
 	cmds "github.com/ipfs/go-ipfs/commands"
 	core "github.com/ipfs/go-ipfs/core"
 	dag "github.com/ipfs/go-ipfs/merkledag"
+	dagutils "github.com/ipfs/go-ipfs/merkledag/utils"
 	path "github.com/ipfs/go-ipfs/path"
 	ft "github.com/ipfs/go-ipfs/unixfs"
 	u "github.com/ipfs/go-ipfs/util"
@@ -453,7 +454,9 @@ This removes the link named foo from the hash in $FOO_BAR and returns the
 resulting object hash.
 `,
 	},
-	Options: []cmds.Option{},
+	Options: []cmds.Option{
+		cmds.BoolOption("create", "p", "create intermediate directories on add-link"),
+	},
 	Arguments: []cmds.Argument{
 		cmds.StringArg("root", true, false, "the hash of the node to modify"),
 		cmds.StringArg("command", true, false, "the operation to perform"),
@@ -467,9 +470,13 @@ resulting object hash.
 			return
 		}
 
-		rhash := key.B58KeyDecode(req.Arguments()[0])
+		rootarg := req.Arguments()[0]
+		if strings.HasPrefix(rootarg, "/ipfs/") {
+			rootarg = rootarg[6:]
+		}
+		rhash := key.B58KeyDecode(rootarg)
 		if rhash == "" {
-			res.SetError(fmt.Errorf("incorrectly formatted root hash"), cmds.ErrNormal)
+			res.SetError(fmt.Errorf("incorrectly formatted root hash: %s", req.Arguments()[0]), cmds.ErrNormal)
 			return
 		}
 
@@ -580,19 +587,18 @@ func rmLinkCaller(req cmds.Request, root *dag.Node) (key.Key, error) {
 		return "", err
 	}
 
-	name := req.Arguments()[2]
+	path := req.Arguments()[2]
 
-	err = root.RemoveNodeLink(name)
+	e := dagutils.NewDagEditor(nd.DAG, root)
+
+	err = e.RmLink(req.Context(), path)
 	if err != nil {
 		return "", err
 	}
 
-	newkey, err := nd.DAG.Add(root)
-	if err != nil {
-		return "", err
-	}
+	nnode := e.GetNode()
 
-	return newkey, nil
+	return nnode.Key()
 }
 
 func addLinkCaller(req cmds.Request, root *dag.Node) (key.Key, error) {
@@ -608,74 +614,30 @@ func addLinkCaller(req cmds.Request, root *dag.Node) (key.Key, error) {
 	path := req.Arguments()[2]
 	childk := key.B58KeyDecode(req.Arguments()[3])
 
-	parts := strings.Split(path, "/")
-
-	nnode, err := insertNodeAtPath(req.Context(), nd.DAG, root, parts, childk)
+	create, _, err := req.Option("create").Bool()
 	if err != nil {
 		return "", err
 	}
+
+	var createfunc func() *dag.Node
+	if create {
+		createfunc = func() *dag.Node {
+			return &dag.Node{Data: ft.FolderPBData()}
+		}
+	}
+
+	e := dagutils.NewDagEditor(nd.DAG, root)
+
+	err = e.InsertNodeAtPath(req.Context(), path, childk, createfunc)
+	if err != nil {
+		return "", err
+	}
+
+	nnode := e.GetNode()
+
 	return nnode.Key()
 }
 
-func addLink(ctx context.Context, ds dag.DAGService, root *dag.Node, childname string, childk key.Key) (*dag.Node, error) {
-	ctx, cancel := context.WithTimeout(ctx, time.Second*30)
-	childnd, err := ds.Get(ctx, childk)
-	if err != nil {
-		cancel()
-		return nil, err
-	}
-	cancel()
-
-	err = root.AddNodeLinkClean(childname, childnd)
-	if err != nil {
-		return nil, err
-	}
-
-	_, err = ds.Add(root)
-	if err != nil {
-		return nil, err
-	}
-	return root, nil
-}
-
-func insertNodeAtPath(ctx context.Context, ds dag.DAGService, root *dag.Node, path []string, toinsert key.Key) (*dag.Node, error) {
-	if len(path) == 1 {
-		return addLink(ctx, ds, root, path[0], toinsert)
-	}
-
-	child, err := root.GetNodeLink(path[0])
-	if err != nil {
-		return nil, err
-	}
-
-	nd, err := child.GetNode(ctx, ds)
-	if err != nil {
-		return nil, err
-	}
-
-	ndprime, err := insertNodeAtPath(ctx, ds, nd, path[1:], toinsert)
-	if err != nil {
-		return nil, err
-	}
-
-	err = root.RemoveNodeLink(path[0])
-	if err != nil {
-		return nil, err
-	}
-
-	err = root.AddNodeLinkClean(path[0], ndprime)
-	if err != nil {
-		return nil, err
-	}
-
-	_, err = ds.Add(root)
-	if err != nil {
-		return nil, err
-	}
-
-	return root, nil
-}
-
 func nodeFromTemplate(template string) (*dag.Node, error) {
 	switch template {
 	case "unixfs-dir":

merkledag/coding.go

@@ -37,16 +37,6 @@ func (n *Node) Unmarshal(encoded []byte) error {
 	return nil
 }
 
-// MarshalTo encodes a *Node instance into a given byte slice.
-// The conversion uses an intermediate PBNode.
-func (n *Node) MarshalTo(encoded []byte) error {
-	pbn := n.getPBNode()
-	if _, err := pbn.MarshalTo(encoded); err != nil {
-		return fmt.Errorf("Marshal failed. %v", err)
-	}
-	return nil
-}
-
 // Marshal encodes a *Node instance into a new byte slice.
 // The conversion uses an intermediate PBNode.
 func (n *Node) Marshal() ([]byte, error) {
@@ -82,7 +72,7 @@ func (n *Node) Encoded(force bool) ([]byte, error) {
 		var err error
 		n.encoded, err = n.Marshal()
 		if err != nil {
-			return []byte{}, err
+			return nil, err
 		}
 		n.cached = u.Hash(n.encoded)
 	}

merkledag/merkledag_test.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"strings"
 	"sync"
 	"testing"
 
@@ -221,3 +222,70 @@ func runBatchFetchTest(t *testing.T, read io.Reader) {
 
 	wg.Wait()
 }
+
+func TestRecursiveAdd(t *testing.T) {
+	a := &Node{Data: []byte("A")}
+	b := &Node{Data: []byte("B")}
+	c := &Node{Data: []byte("C")}
+	d := &Node{Data: []byte("D")}
+	e := &Node{Data: []byte("E")}
+
+	err := a.AddNodeLink("blah", b)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = b.AddNodeLink("foo", c)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = b.AddNodeLink("bar", d)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = d.AddNodeLink("baz", e)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	dsp := getDagservAndPinner(t)
+	err = dsp.ds.AddRecursive(a)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	assertCanGet(t, dsp.ds, a)
+	assertCanGet(t, dsp.ds, b)
+	assertCanGet(t, dsp.ds, c)
+	assertCanGet(t, dsp.ds, d)
+	assertCanGet(t, dsp.ds, e)
+}
+
+func assertCanGet(t *testing.T, ds DAGService, n *Node) {
+	k, err := n.Key()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	_, err = ds.Get(context.TODO(), k)
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestCantGet(t *testing.T) {
+	dsp := getDagservAndPinner(t)
+	a := &Node{Data: []byte("A")}
+
+	k, err := a.Key()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	_, err = dsp.ds.Get(context.TODO(), k)
+	if !strings.Contains(err.Error(), "not found") {
+		t.Fatal("expected err not found, got: ", err)
+	}
+}

merkledag/node.go

@@ -129,13 +129,23 @@ func (n *Node) AddRawLink(name string, l *Link) error {
 // Remove a link on this node by the given name
 func (n *Node) RemoveNodeLink(name string) error {
 	n.encoded = nil
-	for i, l := range n.Links {
-		if l.Name == name {
-			n.Links = append(n.Links[:i], n.Links[i+1:]...)
-			return nil
+	good := make([]*Link, 0, len(n.Links))
+	var found bool
+
+	for _, l := range n.Links {
+		if l.Name != name {
+			good = append(good, l)
+		} else {
+			found = true
 		}
 	}
-	return ErrNotFound
+	n.Links = good
+
+	if !found {
+		return ErrNotFound
+	}
+
+	return nil
 }
 
 // Return a copy of the link with given name
@@ -153,6 +163,15 @@ func (n *Node) GetNodeLink(name string) (*Link, error) {
 	return nil, ErrNotFound
 }
 
+func (n *Node) GetLinkedNode(ctx context.Context, ds DAGService, name string) (*Node, error) {
+	lnk, err := n.GetNodeLink(name)
+	if err != nil {
+		return nil, err
+	}
+
+	return lnk.GetNode(ctx, ds)
+}
+
 // Copy returns a copy of the node.
 // NOTE: does not make copies of Node objects in the links.
 func (n *Node) Copy() *Node {

merkledag/node_test.go

@@ -0,0 +1,54 @@
+package merkledag
+
+import (
+	"testing"
+)
+
+func TestRemoveLink(t *testing.T) {
+	nd := &Node{
+		Links: []*Link{
+			&Link{Name: "a"},
+			&Link{Name: "b"},
+			&Link{Name: "a"},
+			&Link{Name: "a"},
+			&Link{Name: "c"},
+			&Link{Name: "a"},
+		},
+	}
+
+	err := nd.RemoveNodeLink("a")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if len(nd.Links) != 2 {
+		t.Fatal("number of links incorrect")
+	}
+
+	if nd.Links[0].Name != "b" {
+		t.Fatal("link order wrong")
+	}
+
+	if nd.Links[1].Name != "c" {
+		t.Fatal("link order wrong")
+	}
+
+	// should fail
+	err = nd.RemoveNodeLink("a")
+	if err != ErrNotFound {
+		t.Fatal("should have failed to remove link")
+	}
+
+	// ensure nothing else got touched
+	if len(nd.Links) != 2 {
+		t.Fatal("number of links incorrect")
+	}
+
+	if nd.Links[0].Name != "b" {
+		t.Fatal("link order wrong")
+	}
+
+	if nd.Links[1].Name != "c" {
+		t.Fatal("link order wrong")
+	}
+}

merkledag/utils/utils.go

@@ -0,0 +1,155 @@
+package dagutils
+
+import (
+	"errors"
+	"strings"
+
+	context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
+
+	key "github.com/ipfs/go-ipfs/blocks/key"
+	dag "github.com/ipfs/go-ipfs/merkledag"
+)
+
+type Editor struct {
+	root *dag.Node
+	ds   dag.DAGService
+}
+
+func NewDagEditor(ds dag.DAGService, root *dag.Node) *Editor {
+	return &Editor{
+		root: root,
+		ds:   ds,
+	}
+}
+
+func (e *Editor) GetNode() *dag.Node {
+	return e.root.Copy()
+}
+
+func (e *Editor) AddLink(ctx context.Context, childname string, childk key.Key) error {
+	nd, err := addLink(ctx, e.ds, e.root, childname, childk)
+	if err != nil {
+		return err
+	}
+	e.root = nd
+	return nil
+}
+
+func addLink(ctx context.Context, ds dag.DAGService, root *dag.Node, childname string, childk key.Key) (*dag.Node, error) {
+	if childname == "" {
+		return nil, errors.New("cannot create link with no name!")
+	}
+
+	childnd, err := ds.Get(ctx, childk)
+	if err != nil {
+		return nil, err
+	}
+
+	// ensure no link with that name already exists
+	_ = root.RemoveNodeLink(childname) // ignore error, only option is ErrNotFound
+
+	err = root.AddNodeLinkClean(childname, childnd)
+	if err != nil {
+		return nil, err
+	}
+
+	_, err = ds.Add(root)
+	if err != nil {
+		return nil, err
+	}
+	return root, nil
+}
+
+func (e *Editor) InsertNodeAtPath(ctx context.Context, path string, toinsert key.Key, create func() *dag.Node) error {
+	splpath := strings.Split(path, "/")
+	nd, err := insertNodeAtPath(ctx, e.ds, e.root, splpath, toinsert, create)
+	if err != nil {
+		return err
+	}
+	e.root = nd
+	return nil
+}
+
+func insertNodeAtPath(ctx context.Context, ds dag.DAGService, root *dag.Node, path []string, toinsert key.Key, create func() *dag.Node) (*dag.Node, error) {
+	if len(path) == 1 {
+		return addLink(ctx, ds, root, path[0], toinsert)
+	}
+
+	nd, err := root.GetLinkedNode(ctx, ds, path[0])
+	if err != nil {
+		// if 'create' is true, we create directories on the way down as needed
+		if err == dag.ErrNotFound && create != nil {
+			nd = create()
+		} else {
+			return nil, err
+		}
+	}
+
+	ndprime, err := insertNodeAtPath(ctx, ds, nd, path[1:], toinsert, create)
+	if err != nil {
+		return nil, err
+	}
+
+	_ = root.RemoveNodeLink(path[0])
+	err = root.AddNodeLinkClean(path[0], ndprime)
+	if err != nil {
+		return nil, err
+	}
+
+	_, err = ds.Add(root)
+	if err != nil {
+		return nil, err
+	}
+
+	return root, nil
+}
+
+func (e *Editor) RmLink(ctx context.Context, path string) error {
+	splpath := strings.Split(path, "/")
+	nd, err := rmLink(ctx, e.ds, e.root, splpath)
+	if err != nil {
+		return err
+	}
+	e.root = nd
+	return nil
+}
+
+func rmLink(ctx context.Context, ds dag.DAGService, root *dag.Node, path []string) (*dag.Node, error) {
+	if len(path) == 1 {
+		// base case, remove node in question
+		err := root.RemoveNodeLink(path[0])
+		if err != nil {
+			return nil, err
+		}
+
+		_, err = ds.Add(root)
+		if err != nil {
+			return nil, err
+		}
+
+		return root, nil
+	}
+
+	nd, err := root.GetLinkedNode(ctx, ds, path[0])
+	if err != nil {
+		return nil, err
+	}
+
+	nnode, err := rmLink(ctx, ds, nd, path[1:])
+	if err != nil {
+		return nil, err
+	}
+
+	_ = root.RemoveNodeLink(path[0])
+	err = root.AddNodeLinkClean(path[0], nnode)
+	if err != nil {
+		return nil, err
+	}
+
+	_, err = ds.Add(root)
+	if err != nil {
+		return nil, err
+	}
+
+	return root, nil
+}

merkledag/utils/utils_test.go

@@ -0,0 +1,124 @@
+package dagutils
+
+import (
+	"strings"
+	"testing"
+
+	key "github.com/ipfs/go-ipfs/blocks/key"
+	dag "github.com/ipfs/go-ipfs/merkledag"
+	mdtest "github.com/ipfs/go-ipfs/merkledag/test"
+
+	context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
+)
+
+func TestAddLink(t *testing.T) {
+	ds := mdtest.Mock(t)
+	fishnode := &dag.Node{
+		Data: []byte("fishcakes!"),
+	}
+
+	fk, err := ds.Add(fishnode)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	nd := new(dag.Node)
+	nnode, err := addLink(context.Background(), ds, nd, "fish", fk)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	fnprime, err := nnode.GetLinkedNode(context.Background(), ds, "fish")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	fnpkey, err := fnprime.Key()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if fnpkey != fk {
+		t.Fatal("wrong child node found!")
+	}
+}
+
+func assertNodeAtPath(t *testing.T, ds dag.DAGService, root *dag.Node, path string, exp key.Key) {
+	parts := strings.Split(path, "/")
+	cur := root
+	for _, e := range parts {
+		nxt, err := cur.GetLinkedNode(context.Background(), ds, e)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		cur = nxt
+	}
+
+	curk, err := cur.Key()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if curk != exp {
+		t.Fatal("node not as expected at end of path")
+	}
+}
+
+func TestInsertNode(t *testing.T) {
+	ds := mdtest.Mock(t)
+	root := new(dag.Node)
+	e := NewDagEditor(ds, root)
+
+	testInsert(t, e, "a", "anodefortesting", false, "")
+	testInsert(t, e, "a/b", "data", false, "")
+	testInsert(t, e, "a/b/c/d/e", "blah", false, "merkledag: not found")
+	testInsert(t, e, "a/b/c/d/e", "foo", true, "")
+	testInsert(t, e, "a/b/c/d/f", "baz", true, "")
+	testInsert(t, e, "a/b/c/d/f", "bar", true, "")
+
+	testInsert(t, e, "", "bar", true, "cannot create link with no name!")
+	testInsert(t, e, "////", "slashes", true, "cannot create link with no name!")
+
+	k, err := e.GetNode().Key()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if k.B58String() != "QmThorWojP6YzLJwDukxiYCoKQSwyrMCvdt4WZ6rPm221t" {
+		t.Fatal("output was different than expected")
+	}
+}
+
+func testInsert(t *testing.T, e *Editor, path, data string, create bool, experr string) {
+	child := &dag.Node{Data: []byte(data)}
+	ck, err := e.ds.Add(child)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var c func() *dag.Node
+	if create {
+		c = func() *dag.Node {
+			return &dag.Node{}
+		}
+	}
+
+	err = e.InsertNodeAtPath(context.TODO(), path, ck, c)
+	if experr != "" {
+		var got string
+		if err != nil {
+			got = err.Error()
+		}
+		if got != experr {
+			t.Fatalf("expected '%s' but got '%s'", experr, got)
+		}
+		return
+	}
+
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	assertNodeAtPath(t, e.ds, e.root, path, ck)
+}

test/sharness/t0051-object.sh

@@ -10,6 +10,22 @@ test_description="Test object command"
 
 test_init_ipfs
 
+test_patch_create_path() {
+	root=$1
+	name=$2
+	target=$3
+
+	test_expect_success "object patch --create works" '
+		PCOUT=$(ipfs object patch --create $root add-link $name $target)
+	'
+
+	test_expect_success "output looks good" '
+		ipfs cat $PCOUT/$name > tpcp_out &&
+		ipfs cat $target > tpcp_exp &&
+		test_cmp tpcp_out tpcp_exp
+	'
+}
+
 test_object_cmd() {
 
 	test_expect_success "'ipfs add testData' succeeds" '
@@ -145,6 +161,25 @@ test_object_cmd() {
 		echo QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn > rmlink_exp &&
 		test_cmp rmlink_exp rmlink_output
 	'
+
+	test_expect_success "multilayer rm-link should work" '
+		ipfs object patch $(cat multi_patch) rm-link a/b/c > multi_link_rm_out
+	'
+
+	test_expect_success "output looks good" '
+		echo "QmZD3r9cZjzU8huNY2JS9TC6n8daDfT8TmE8zBSqG31Wvq" > multi_link_rm_exp &&
+		test_cmp multi_link_rm_out multi_link_rm_exp
+	'
+
+	test_patch_create_path $EMPTY a/b/c $FILE
+
+	test_patch_create_path $EMPTY a $FILE
+
+	test_patch_create_path $EMPTY a/b/b/b/b $FILE
+
+	test_expect_success "create bad path fails" '
+		test_must_fail ipfs object patch --create $EMPTY add-link / $FILE
+	'
 }
 
 # should work offline