opensource/kubo
1
0
Fork 0
mirror of https://github.com/ipfs/kubo.git synced 2025-09-10 11:52:21 +08:00
Code Issues Projects Releases Wiki Activity

rework editor creation and finalization

Browse Source 
License: MIT
Signed-off-by: Jeromy <jeromyj@gmail.com>
This commit is contained in:
Jeromy
2015-11-13 10:19:47 -08:00
parent d8ee7dffc3
commit efac042e82
9 changed files with 113 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
core/commands/add.go
View File

@ -149,13 +149,15 @@ remains to be implemented.
return err
}
if !hash {
// copy intermediary nodes from editor to our actual dagservice
err := fileAdder.WriteOutputTo(n.DAG)
if err != nil {
log.Error("WRITE OUT: ", err)
return err
}
if hash {
return nil
}
// copy intermediary nodes from editor to our actual dagservice
_, err := fileAdder.Finalize(n.DAG)
if err != nil {
log.Error("WRITE OUT: ", err)
return err
}
return fileAdder.PinRoot()

14
core/commands/object.go
View File

@ -599,14 +599,17 @@ func rmLinkCaller(req cmds.Request, root *dag.Node) (key.Key, error) {
path := req.Arguments()[2]
e := dagutils.NewDagEditor(nd.DAG, root)
e := dagutils.NewDagEditor(root, nd.DAG)
err = e.RmLink(req.Context(), path)
if err != nil {
return "", err
}
nnode := e.GetNode()
nnode, err := e.Finalize(nd.DAG)
if err != nil {
return "", err
}
return nnode.Key()
}
@ -636,7 +639,7 @@ func addLinkCaller(req cmds.Request, root *dag.Node) (key.Key, error) {
}
}
e := dagutils.NewDagEditor(nd.DAG, root)
e := dagutils.NewDagEditor(root, nd.DAG)
childnd, err := nd.DAG.Get(req.Context(), childk)
if err != nil {
@ -648,7 +651,10 @@ func addLinkCaller(req cmds.Request, root *dag.Node) (key.Key, error) {
return "", err
}
nnode := e.GetNode()
nnode, err := e.Finalize(nd.DAG)
if err != nil {
return "", err
}
return nnode.Key()
}

10
core/corehttp/gateway_handler.go
View File

@ -342,14 +342,20 @@ func (i *gatewayHandler) putHandler(w http.ResponseWriter, r *http.Request) {
return
}
e := dagutils.NewDagEditor(i.node.DAG, rnode)
e := dagutils.NewDagEditor(rnode, i.node.DAG)
err = e.InsertNodeAtPath(ctx, newPath, newnode, uio.NewEmptyDirectory)
if err != nil {
webError(w, "putHandler: InsertNodeAtPath failed", err, http.StatusInternalServerError)
return
}
newkey, err = e.GetNode().Key()
nnode, err := e.Finalize(i.node.DAG)
if err != nil {
webError(w, "putHandler: could not get node", err, http.StatusInternalServerError)
return
}
newkey, err = nnode.Key()
if err != nil {
webError(w, "putHandler: could not get key of edited node", err, http.StatusInternalServerError)
return

36
core/coreunix/add.go
View File

@ -20,7 +20,7 @@ import (
"github.com/ipfs/go-ipfs/commands/files"
core "github.com/ipfs/go-ipfs/core"
merkledag "github.com/ipfs/go-ipfs/merkledag"
dag "github.com/ipfs/go-ipfs/merkledag"
unixfs "github.com/ipfs/go-ipfs/unixfs"
logging "github.com/ipfs/go-ipfs/vendor/QmQg1J6vikuXF9oDvm4wpdeAUvvkVEKW1EYDw9HhTMnP2b/go-log"
)
@ -63,7 +63,7 @@ type AddedObject struct {
}
func NewAdder(ctx context.Context, n *core.IpfsNode, out chan interface{}) *Adder {
e := dagutils.NewDagEditor(NewMemoryDagService(), newDirNode())
e := dagutils.NewDagEditor(newDirNode(), nil)
return &Adder{
ctx: ctx,
node: n,
@ -90,11 +90,11 @@ type Adder struct {
Trickle bool
Wrap bool
Chunker string
root *merkledag.Node
root *dag.Node
}
// Perform the actual add & pin locally, outputting results to reader
func (params Adder) add(reader io.Reader) (*merkledag.Node, error) {
func (params Adder) add(reader io.Reader) (*dag.Node, error) {
chnk, err := chunk.FromString(reader, params.Chunker)
if err != nil {
return nil, err
@ -112,7 +112,7 @@ func (params Adder) add(reader io.Reader) (*merkledag.Node, error) {
)
}
func (params *Adder) RootNode() (*merkledag.Node, error) {
func (params *Adder) RootNode() (*dag.Node, error) {
// for memoizing
if params.root != nil {
return params.root, nil
@ -153,8 +153,8 @@ func (params *Adder) PinRoot() error {
return params.node.Pinning.Flush()
}
func (params *Adder) WriteOutputTo(DAG merkledag.DAGService) error {
return params.editor.WriteOutputTo(DAG)
func (params *Adder) Finalize(DAG dag.DAGService) (*dag.Node, error) {
return params.editor.Finalize(DAG)
}
// Add builds a merkledag from the a reader, pinning all objects to the local
@ -212,7 +212,7 @@ func AddR(n *core.IpfsNode, root string) (key string, err error) {
// to preserve the filename.
// Returns the path of the added file ("<dir hash>/filename"), the DAG node of
// the directory, and and error if any.
func AddWrapped(n *core.IpfsNode, r io.Reader, filename string) (string, *merkledag.Node, error) {
func AddWrapped(n *core.IpfsNode, r io.Reader, filename string) (string, *dag.Node, error) {
file := files.NewReaderFile(filename, filename, ioutil.NopCloser(r), nil)
dir := files.NewSliceFile("", "", []files.File{file})
fileAdder := NewAdder(n.Context(), n, nil)
@ -230,7 +230,7 @@ func AddWrapped(n *core.IpfsNode, r io.Reader, filename string) (string, *merkle
return gopath.Join(k.String(), filename), dagnode, nil
}
func (params *Adder) addNode(node *merkledag.Node, path string) error {
func (params *Adder) addNode(node *dag.Node, path string) error {
// patch it into the root
if path == "" {
key, err := node.Key()
@ -249,7 +249,7 @@ func (params *Adder) addNode(node *merkledag.Node, path string) error {
}
// Add the given file while respecting the params.
func (params *Adder) AddFile(file files.File) (*merkledag.Node, error) {
func (params *Adder) AddFile(file files.File) (*dag.Node, error) {
switch {
case files.IsHidden(file) && !params.Hidden:
log.Debugf("%s is hidden, skipping", file.FileName())
@ -265,7 +265,7 @@ func (params *Adder) AddFile(file files.File) (*merkledag.Node, error) {
return nil, err
}
dagnode := &merkledag.Node{Data: sdata}
dagnode := &dag.Node{Data: sdata}
_, err = params.node.DAG.Add(dagnode)
if err != nil {
return nil, err
@ -294,7 +294,7 @@ func (params *Adder) AddFile(file files.File) (*merkledag.Node, error) {
return dagnode, err
}
func (params *Adder) addDir(dir files.File) (*merkledag.Node, error) {
func (params *Adder) addDir(dir files.File) (*dag.Node, error) {
tree := newDirNode()
log.Infof("adding directory: %s", dir.FileName())
@ -334,7 +334,7 @@ func (params *Adder) addDir(dir files.File) (*merkledag.Node, error) {
}
// outputDagnode sends dagnode info over the output channel
func outputDagnode(out chan interface{}, name string, dn *merkledag.Node) error {
func outputDagnode(out chan interface{}, name string, dn *dag.Node) error {
if out == nil {
return nil
}
@ -352,20 +352,20 @@ func outputDagnode(out chan interface{}, name string, dn *merkledag.Node) error
return nil
}
func NewMemoryDagService() merkledag.DAGService {
func NewMemoryDagService() dag.DAGService {
// build mem-datastore for editor's intermediary nodes
bs := bstore.NewBlockstore(syncds.MutexWrap(ds.NewMapDatastore()))
bsrv := bserv.New(bs, offline.Exchange(bs))
return merkledag.NewDAGService(bsrv)
return dag.NewDAGService(bsrv)
}
// TODO: generalize this to more than unix-fs nodes.
func newDirNode() *merkledag.Node {
return &merkledag.Node{Data: unixfs.FolderPBData()}
func newDirNode() *dag.Node {
return &dag.Node{Data: unixfs.FolderPBData()}
}
// from core/commands/object.go
func getOutput(dagnode *merkledag.Node) (*Object, error) {
func getOutput(dagnode *dag.Node) (*Object, error) {
key, err := dagnode.Key()
if err != nil {
return nil, err

4
merkledag/node.go
View File

@ -9,6 +9,8 @@ import (
key "github.com/ipfs/go-ipfs/blocks/key"
)
var ErrLinkNotFound = fmt.Errorf("no link by that name")
// Node represents a node in the IPFS Merkle DAG.
// nodes have opaque data and a set of navigable links.
type Node struct {
@ -160,7 +162,7 @@ func (n *Node) GetNodeLink(name string) (*Link, error) {
}, nil
}
}
return nil, ErrNotFound
return nil, ErrLinkNotFound
}
func (n *Node) GetLinkedNode(ctx context.Context, ds DAGService, name string) (*Node, error) {

5
merkledag/utils/diff.go
View File

@ -37,7 +37,7 @@ func (c *Change) String() string {
}
func ApplyChange(ctx context.Context, ds dag.DAGService, nd *dag.Node, cs []*Change) (*dag.Node, error) {
e := NewDagEditor(ds, nd)
e := NewDagEditor(nd, ds)
for _, c := range cs {
switch c.Type {
case Add:
@ -71,7 +71,8 @@ func ApplyChange(ctx context.Context, ds dag.DAGService, nd *dag.Node, cs []*Cha
}
}
}
return e.GetNode(), nil
return e.Finalize(ds)
}
func Diff(ctx context.Context, ds dag.DAGService, a, b *dag.Node) []*Change {

76
merkledag/utils/utils.go
View File

@ -4,20 +4,41 @@ import (
"errors"
"strings"
ds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
syncds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/sync"
context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
bstore "github.com/ipfs/go-ipfs/blocks/blockstore"
bserv "github.com/ipfs/go-ipfs/blockservice"
offline "github.com/ipfs/go-ipfs/exchange/offline"
dag "github.com/ipfs/go-ipfs/merkledag"
)
type Editor struct {
root *dag.Node
ds dag.DAGService
// tmp is a temporary in memory (for now) dagstore for all of the
// intermediary nodes to be stored in
tmp dag.DAGService
// src is the dagstore with *all* of the data on it, it is used to pull
// nodes from for modification (nil is a valid value)
src dag.DAGService
}
func NewDagEditor(ds dag.DAGService, root *dag.Node) *Editor {
func NewMemoryDagService() dag.DAGService {
// build mem-datastore for editor's intermediary nodes
bs := bstore.NewBlockstore(syncds.MutexWrap(ds.NewMapDatastore()))
bsrv := bserv.New(bs, offline.Exchange(bs))
return dag.NewDAGService(bsrv)
}
// root is the node to be modified, source is the dagstore to pull nodes from (optional)
func NewDagEditor(root *dag.Node, source dag.DAGService) *Editor {
return &Editor{
root: root,
ds: ds,
tmp: NewMemoryDagService(),
src: source,
}
}
@ -26,7 +47,7 @@ func (e *Editor) GetNode() *dag.Node {
}
func (e *Editor) GetDagService() dag.DAGService {
return e.ds
return e.tmp
}
func addLink(ctx context.Context, ds dag.DAGService, root *dag.Node, childname string, childnd *dag.Node) (*dag.Node, error) {
@ -57,7 +78,7 @@ func addLink(ctx context.Context, ds dag.DAGService, root *dag.Node, childname s
func (e *Editor) InsertNodeAtPath(ctx context.Context, path string, toinsert *dag.Node, create func() *dag.Node) error {
splpath := strings.Split(path, "/")
nd, err := insertNodeAtPath(ctx, e.ds, e.root, splpath, toinsert, create)
nd, err := e.insertNodeAtPath(ctx, e.root, splpath, toinsert, create)
if err != nil {
return err
}
@ -65,27 +86,32 @@ func (e *Editor) InsertNodeAtPath(ctx context.Context, path string, toinsert *da
return nil
}
func insertNodeAtPath(ctx context.Context, ds dag.DAGService, root *dag.Node, path []string, toinsert *dag.Node, create func() *dag.Node) (*dag.Node, error) {
func (e *Editor) insertNodeAtPath(ctx context.Context, root *dag.Node, path []string, toinsert *dag.Node, create func() *dag.Node) (*dag.Node, error) {
if len(path) == 1 {
return addLink(ctx, ds, root, path[0], toinsert)
return addLink(ctx, e.tmp, root, path[0], toinsert)
}
nd, err := root.GetLinkedNode(ctx, ds, path[0])
nd, err := root.GetLinkedNode(ctx, e.tmp, path[0])
if err != nil {
// if 'create' is true, we create directories on the way down as needed
if err == dag.ErrNotFound && create != nil {
if err == dag.ErrLinkNotFound && create != nil {
nd = create()
} else {
err = nil // no longer an error case
} else if err == dag.ErrNotFound {
nd, err = root.GetLinkedNode(ctx, e.src, path[0])
}
if err != nil {
return nil, err
}
}
ndprime, err := insertNodeAtPath(ctx, ds, nd, path[1:], toinsert, create)
ndprime, err := e.insertNodeAtPath(ctx, nd, path[1:], toinsert, create)
if err != nil {
return nil, err
}
_ = ds.Remove(root)
_ = e.tmp.Remove(root)
_ = root.RemoveNodeLink(path[0])
err = root.AddNodeLinkClean(path[0], ndprime)
@ -93,7 +119,7 @@ func insertNodeAtPath(ctx context.Context, ds dag.DAGService, root *dag.Node, pa
return nil, err
}
_, err = ds.Add(root)
_, err = e.tmp.Add(root)
if err != nil {
return nil, err
}
@ -103,7 +129,7 @@ func insertNodeAtPath(ctx context.Context, ds dag.DAGService, root *dag.Node, pa
func (e *Editor) RmLink(ctx context.Context, path string) error {
splpath := strings.Split(path, "/")
nd, err := rmLink(ctx, e.ds, e.root, splpath)
nd, err := e.rmLink(ctx, e.root, splpath)
if err != nil {
return err
}
@ -111,7 +137,7 @@ func (e *Editor) RmLink(ctx context.Context, path string) error {
return nil
}
func rmLink(ctx context.Context, ds dag.DAGService, root *dag.Node, path []string) (*dag.Node, error) {
func (e *Editor) rmLink(ctx context.Context, root *dag.Node, path []string) (*dag.Node, error) {
if len(path) == 1 {
// base case, remove node in question
err := root.RemoveNodeLink(path[0])
@ -119,7 +145,7 @@ func rmLink(ctx context.Context, ds dag.DAGService, root *dag.Node, path []strin
return nil, err
}
_, err = ds.Add(root)
_, err = e.tmp.Add(root)
if err != nil {
return nil, err
}
@ -127,17 +153,21 @@ func rmLink(ctx context.Context, ds dag.DAGService, root *dag.Node, path []strin
return root, nil
}
nd, err := root.GetLinkedNode(ctx, ds, path[0])
nd, err := root.GetLinkedNode(ctx, e.tmp, path[0])
if err == dag.ErrNotFound {
nd, err = root.GetLinkedNode(ctx, e.src, path[0])
}
if err != nil {
return nil, err
}
nnode, err := rmLink(ctx, ds, nd, path[1:])
nnode, err := e.rmLink(ctx, nd, path[1:])
if err != nil {
return nil, err
}
_ = ds.Remove(root)
_ = e.tmp.Remove(root)
_ = root.RemoveNodeLink(path[0])
err = root.AddNodeLinkClean(path[0], nnode)
@ -145,7 +175,7 @@ func rmLink(ctx context.Context, ds dag.DAGService, root *dag.Node, path []strin
return nil, err
}
_, err = ds.Add(root)
_, err = e.tmp.Add(root)
if err != nil {
return nil, err
}
@ -153,8 +183,10 @@ func rmLink(ctx context.Context, ds dag.DAGService, root *dag.Node, path []strin
return root, nil
}
func (e *Editor) WriteOutputTo(ds dag.DAGService) error {
return copyDag(e.GetNode(), e.ds, ds)
func (e *Editor) Finalize(ds dag.DAGService) (*dag.Node, error) {
nd := e.GetNode()
err := copyDag(nd, e.tmp, ds)
return nd, err
}
func copyDag(nd *dag.Node, from, to dag.DAGService) error {

11
merkledag/utils/utils_test.go
View File

@ -66,13 +66,12 @@ func assertNodeAtPath(t *testing.T, ds dag.DAGService, root *dag.Node, path stri
}
func TestInsertNode(t *testing.T) {
ds := mdtest.Mock()
root := new(dag.Node)
e := NewDagEditor(ds, root)
e := NewDagEditor(root, nil)
testInsert(t, e, "a", "anodefortesting", false, "")
testInsert(t, e, "a/b", "data", false, "")
testInsert(t, e, "a/b/c/d/e", "blah", false, "merkledag: not found")
testInsert(t, e, "a/b/c/d/e", "blah", false, "no link by that name")
testInsert(t, e, "a/b/c/d/e", "foo", true, "")
testInsert(t, e, "a/b/c/d/f", "baz", true, "")
testInsert(t, e, "a/b/c/d/f", "bar", true, "")
@ -92,7 +91,7 @@ func TestInsertNode(t *testing.T) {
func testInsert(t *testing.T, e *Editor, path, data string, create bool, experr string) {
child := &dag.Node{Data: []byte(data)}
ck, err := e.ds.Add(child)
ck, err := e.tmp.Add(child)
if err != nil {
t.Fatal(err)
}
@ -117,8 +116,8 @@ func testInsert(t *testing.T, e *Editor, path, data string, create bool, experr
}
if err != nil {
t.Fatal(err)
t.Fatal(err, path, data, create, experr)
}
assertNodeAtPath(t, e.ds, e.root, path, ck)
assertNodeAtPath(t, e.tmp, e.root, path, ck)
}

12
tar/format.go
View File

@ -46,7 +46,7 @@ func ImportTar(r io.Reader, ds dag.DAGService) (*dag.Node, error) {
root := new(dag.Node)
root.Data = []byte("ipfs/tar")
e := dagutil.NewDagEditor(ds, root)
e := dagutil.NewDagEditor(root, ds)
for {
h, err := tr.Next()
@ -91,13 +91,7 @@ func ImportTar(r io.Reader, ds dag.DAGService) (*dag.Node, error) {
}
}
root = e.GetNode()
_, err = ds.Add(root)
if err != nil {
return nil, err
}
return root, nil
return e.Finalize(ds)
}
// adds a '-' to the beginning of each path element so we can use 'data' as a
@ -178,7 +172,7 @@ func (tr *tarReader) Read(b []byte) (int, error) {
tr.hdrBuf = bytes.NewReader(headerNd.Data)
dataNd, err := headerNd.GetLinkedNode(tr.ctx, tr.ds, "data")
if err != nil && err != dag.ErrNotFound {
if err != nil && err != dag.ErrLinkNotFound {
return 0, err
}