add-only-hash no longer stores entirety of everything in memory

License: MIT
Signed-off-by: Jeromy <jeromyj@gmail.com>

core/commands/add.go

@@ -6,11 +6,16 @@ import (
 	"path"
 
 	"github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/cheggaaa/pb"
+	ds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
+	syncds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/sync"
 	cxt "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
 
+	bstore "github.com/ipfs/go-ipfs/blocks/blockstore"
+	bserv "github.com/ipfs/go-ipfs/blockservice"
 	cmds "github.com/ipfs/go-ipfs/commands"
 	files "github.com/ipfs/go-ipfs/commands/files"
 	core "github.com/ipfs/go-ipfs/core"
+	offline "github.com/ipfs/go-ipfs/exchange/offline"
 	importer "github.com/ipfs/go-ipfs/importer"
 	"github.com/ipfs/go-ipfs/importer/chunk"
 	dag "github.com/ipfs/go-ipfs/merkledag"
@@ -103,17 +108,28 @@ remains to be implemented.
 		hidden, _, _ := req.Option(hiddenOptionName).Bool()
 		chunker, _, _ := req.Option(chunkerOptionName).String()
 
+		e := dagutils.NewDagEditor(n.DAG, newDirNode())
 		if hash {
 			nilnode, err := core.NewNode(n.Context(), &core.BuildCfg{
 				//TODO: need this to be true or all files
 				// hashed will be stored in memory!
-				NilRepo: false,
+				NilRepo: true,
 			})
 			if err != nil {
 				res.SetError(err, cmds.ErrNormal)
 				return
 			}
 			n = nilnode
+
+			// build mem-datastore for editor's intermediary nodes
+			bs := bstore.NewBlockstore(syncds.MutexWrap(ds.NewMapDatastore()))
+			bsrv, err := bserv.New(bs, offline.Exchange(bs))
+			if err != nil {
+				res.SetError(err, cmds.ErrNormal)
+				return
+			}
+			memds := dag.NewDAGService(bsrv)
+			e = dagutils.NewDagEditor(memds, newDirNode())
 		}
 
 		outChan := make(chan interface{}, 8)
@@ -122,7 +138,7 @@ remains to be implemented.
 		fileAdder := adder{
 			ctx:      req.Context(),
 			node:     n,
-			editor:   dagutils.NewDagEditor(n.DAG, newDirNode()),
+			editor:   e,
 			out:      outChan,
 			chunker:  chunker,
 			progress: progress,
@@ -318,7 +334,7 @@ func (params *adder) RootNode() (*dag.Node, error) {
 	// if not wrapping, AND one root file, use that hash as root.
 	if !params.wrap && len(r.Links) == 1 {
 		var err error
-		r, err = r.Links[0].GetNode(params.ctx, params.node.DAG)
+		r, err = r.Links[0].GetNode(params.ctx, params.editor.GetDagService())
 		// no need to output, as we've already done so.
 		return r, err
 	}
@@ -330,16 +346,16 @@ func (params *adder) RootNode() (*dag.Node, error) {
 
 func (params *adder) addNode(node *dag.Node, path string) error {
 	// patch it into the root
+	if path == "" {
 		key, err := node.Key()
 		if err != nil {
 			return err
 		}
 
-	if path == "" {
 		path = key.Pretty()
 	}
 
-	if err := params.editor.InsertNodeAtPath(params.ctx, path, key, newDirNode); err != nil {
+	if err := params.editor.InsertNodeAtPath(params.ctx, path, node, newDirNode); err != nil {
 		return err
 	}
 

core/commands/object.go

@@ -623,7 +623,12 @@ func addLinkCaller(req cmds.Request, root *dag.Node) (key.Key, error) {
 
 	e := dagutils.NewDagEditor(nd.DAG, root)
 
-	err = e.InsertNodeAtPath(req.Context(), path, childk, createfunc)
+	childnd, err := nd.DAG.Get(req.Context(), childk)
+	if err != nil {
+		return "", err
+	}
+
+	err = e.InsertNodeAtPath(req.Context(), path, childnd, createfunc)
 	if err != nil {
 		return "", err
 	}

merkledag/utils/utils.go

@@ -6,7 +6,6 @@ import (
 
 	context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
 
-	key "github.com/ipfs/go-ipfs/blocks/key"
 	dag "github.com/ipfs/go-ipfs/merkledag"
 )
 
@@ -26,21 +25,17 @@ func (e *Editor) GetNode() *dag.Node {
 	return e.root.Copy()
 }
 
-func (e *Editor) AddLink(ctx context.Context, childname string, childk key.Key) error {
-	nd, err := addLink(ctx, e.ds, e.root, childname, childk)
-	if err != nil {
-		return err
-	}
-	e.root = nd
-	return nil
+func (e *Editor) GetDagService() dag.DAGService {
+	return e.ds
 }
 
-func addLink(ctx context.Context, ds dag.DAGService, root *dag.Node, childname string, childk key.Key) (*dag.Node, error) {
+func addLink(ctx context.Context, ds dag.DAGService, root *dag.Node, childname string, childnd *dag.Node) (*dag.Node, error) {
 	if childname == "" {
 		return nil, errors.New("cannot create link with no name!")
 	}
 
-	childnd, err := ds.Get(ctx, childk)
+	// ensure that the node we are adding is in the dagservice
+	_, err := ds.Add(childnd)
 	if err != nil {
 		return nil, err
 	}
@@ -58,7 +53,7 @@ func addLink(ctx context.Context, ds dag.DAGService, root *dag.Node, childname s
 	return root, nil
 }
 
-func (e *Editor) InsertNodeAtPath(ctx context.Context, path string, toinsert key.Key, create func() *dag.Node) error {
+func (e *Editor) InsertNodeAtPath(ctx context.Context, path string, toinsert *dag.Node, create func() *dag.Node) error {
 	splpath := strings.Split(path, "/")
 	nd, err := insertNodeAtPath(ctx, e.ds, e.root, splpath, toinsert, create)
 	if err != nil {
@@ -68,7 +63,7 @@ func (e *Editor) InsertNodeAtPath(ctx context.Context, path string, toinsert key
 	return nil
 }
 
-func insertNodeAtPath(ctx context.Context, ds dag.DAGService, root *dag.Node, path []string, toinsert key.Key, create func() *dag.Node) (*dag.Node, error) {
+func insertNodeAtPath(ctx context.Context, ds dag.DAGService, root *dag.Node, path []string, toinsert *dag.Node, create func() *dag.Node) (*dag.Node, error) {
 	if len(path) == 1 {
 		return addLink(ctx, ds, root, path[0], toinsert)
 	}

merkledag/utils/utils_test.go

@@ -23,7 +23,7 @@ func TestAddLink(t *testing.T) {
 	}
 
 	nd := new(dag.Node)
-	nnode, err := addLink(context.Background(), ds, nd, "fish", fk)
+	nnode, err := addLink(context.Background(), ds, nd, "fish", fishnode)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -104,7 +104,7 @@ func testInsert(t *testing.T, e *Editor, path, data string, create bool, experr
 		}
 	}
 
-	err = e.InsertNodeAtPath(context.Background(), path, ck, c)
+	err = e.InsertNodeAtPath(context.Background(), path, child, c)
 	if experr != "" {
 		var got string
 		if err != nil {