opensource/kubo
1
0
Fork 0
mirror of https://github.com/ipfs/kubo.git synced 2025-06-24 05:59:55 +08:00
Code Issues Projects Releases Wiki Activity

Merge pull request #2359 from mildred/ipld

Browse Source 
Simplify merkledag package interface to ease IPLD inclusion
This commit is contained in:
Jeromy Johnson
2016-03-04 12:02:37 -08:00
parent 5f3945c9b7 683b81760e
commit 91c6f0f3db
16 changed files with 178 additions and 212 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
core/commands/ls.go
View File

@ -77,12 +77,13 @@ Displays the links an IPFS or IPNS object(s) contains, with the following format
Links: make([]LsLink, len(dagnode.Links)),
}
for j, link := range dagnode.Links {
link.Node, err = link.GetNode(req.Context(), node.DAG)
var linkNode *merkledag.Node
linkNode, err = link.GetNode(req.Context(), node.DAG)
if err != nil {
res.SetError(err, cmds.ErrNormal)
return
}
d, err := unixfs.FromBytes(link.Node.Data)
d, err := unixfs.FromBytes(linkNode.Data)
if err != nil {
res.SetError(err, cmds.ErrNormal)
return

2
core/commands/object/object.go
View File

@ -486,7 +486,7 @@ func objectPut(n *core.IpfsNode, input io.Reader, encoding string) (*Object, err
}
case objectEncodingProtobuf:
dagnode, err = dag.Decoded(data)
dagnode, err = dag.DecodeProtobuf(data)
case objectEncodingXML:
node := new(Node)

6
core/commands/unixfs/ls.go
View File

@ -12,6 +12,7 @@ import (
path "github.com/ipfs/go-ipfs/path"
unixfs "github.com/ipfs/go-ipfs/unixfs"
unixfspb "github.com/ipfs/go-ipfs/unixfs/pb"
merkledag "github.com/ipfs/go-ipfs/merkledag"
)
type LsLink struct {
@ -104,12 +105,13 @@ size is the IPFS link size.
links := make([]LsLink, len(merkleNode.Links))
output.Objects[hash].Links = links
for i, link := range merkleNode.Links {
link.Node, err = link.GetNode(ctx, node.DAG)
var linkNode *merkledag.Node
linkNode, err = link.GetNode(ctx, node.DAG)
if err != nil {
res.SetError(err, cmds.ErrNormal)
return
}
d, err := unixfs.FromBytes(link.Node.Data)
d, err := unixfs.FromBytes(linkNode.Data)
if err != nil {
res.SetError(err, cmds.ErrNormal)
return

12
core/corehttp/gateway_handler.go
View File

@ -453,16 +453,20 @@ func (i *gatewayHandler) deleteHandler(w http.ResponseWriter, r *http.Request) {
}
newnode := pathNodes[len(pathNodes)-1]
for i := len(pathNodes) - 2; i >= 0; i-- {
newnode, err = pathNodes[i].UpdateNodeLink(components[i], newnode)
for j := len(pathNodes) - 2; j >= 0; j-- {
if _, err := i.node.DAG.Add(newnode); err != nil {
webError(w, "Could not add node", err, http.StatusInternalServerError)
return
}
newnode, err = pathNodes[j].UpdateNodeLink(components[j], newnode)
if err != nil {
webError(w, "Could not update node links", err, http.StatusInternalServerError)
return
}
}
if err := i.node.DAG.AddRecursive(newnode); err != nil {
webError(w, "Could not add recursively new node", err, http.StatusInternalServerError)
if _, err := i.node.DAG.Add(newnode); err != nil {
webError(w, "Could not add root node", err, http.StatusInternalServerError)
return
}

17
core/corehttp/gateway_test.go
View File

@ -181,7 +181,12 @@ func TestIPNSHostnameRedirect(t *testing.T) {
t.Fatal(err)
}
err = n.DAG.AddRecursive(dagn1)
_, err = n.DAG.Add(dagn2)
if err != nil {
t.Fatal(err)
}
_, err = n.DAG.Add(dagn1)
if err != nil {
t.Fatal(err)
}
@ -266,7 +271,15 @@ func TestIPNSHostnameBacklinks(t *testing.T) {
t.Fatal(err)
}
err = n.DAG.AddRecursive(dagn1)
_, err = n.DAG.Add(dagn3)
if err != nil {
t.Fatal(err)
}
_, err = n.DAG.Add(dagn2)
if err != nil {
t.Fatal(err)
}
_, err = n.DAG.Add(dagn1)
if err != nil {
t.Fatal(err)
}

14
merkledag/coding.go
View File

@ -13,9 +13,9 @@ import (
// for now, we use a PBNode intermediate thing.
// because native go objects are nice.
// Unmarshal decodes raw data into a *Node instance.
// unmarshal decodes raw data into a *Node instance.
// The conversion uses an intermediate PBNode.
func (n *Node) Unmarshal(encoded []byte) error {
func (n *Node) unmarshal(encoded []byte) error {
var pbn pb.PBNode
if err := pbn.Unmarshal(encoded); err != nil {
return fmt.Errorf("Unmarshal failed. %v", err)
@ -27,7 +27,7 @@ func (n *Node) Unmarshal(encoded []byte) error {
n.Links[i] = &Link{Name: l.GetName(), Size: l.GetTsize()}
h, err := mh.Cast(l.GetHash())
if err != nil {
return fmt.Errorf("Link hash is not valid multihash. %v", err)
return fmt.Errorf("Link hash #%d is not valid multihash. %v", i, err)
}
n.Links[i].Hash = h
}
@ -68,9 +68,9 @@ func (n *Node) getPBNode() *pb.PBNode {
return pbn
}
// Encoded returns the encoded raw data version of a Node instance.
// EncodeProtobuf returns the encoded raw data version of a Node instance.
// It may use a cached encoded version, unless the force flag is given.
func (n *Node) Encoded(force bool) ([]byte, error) {
func (n *Node) EncodeProtobuf(force bool) ([]byte, error) {
sort.Stable(LinkSlice(n.Links)) // keep links sorted
if n.encoded == nil || force {
var err error
@ -85,9 +85,9 @@ func (n *Node) Encoded(force bool) ([]byte, error) {
}
// Decoded decodes raw data and returns a new Node instance.
func Decoded(encoded []byte) (*Node, error) {
func DecodeProtobuf(encoded []byte) (*Node, error) {
n := new(Node)
err := n.Unmarshal(encoded)
err := n.unmarshal(encoded)
if err != nil {
return nil, fmt.Errorf("incorrectly formatted merkledag node: %s", err)
}

46
merkledag/merkledag.go
View File

@ -18,10 +18,8 @@ var ErrNotFound = fmt.Errorf("merkledag: not found")
// DAGService is an IPFS Merkle DAG service.
type DAGService interface {
Add(*Node) (key.Key, error)
AddRecursive(*Node) error
Get(context.Context, key.Key) (*Node, error)
Remove(*Node) error
RemoveRecursive(*Node) error
// GetDAG returns, in order, all the single leve child
// nodes of the passed in node.
@ -49,7 +47,7 @@ func (n *dagService) Add(nd *Node) (key.Key, error) {
return "", fmt.Errorf("dagService is nil")
}
d, err := nd.Encoded(false)
d, err := nd.EncodeProtobuf(false)
if err != nil {
return "", err
}
@ -68,26 +66,6 @@ func (n *dagService) Batch() *Batch {
return &Batch{ds: n, MaxSize: 8 * 1024 * 1024}
}
// AddRecursive adds the given node and all child nodes to the BlockService
func (n *dagService) AddRecursive(nd *Node) error {
_, err := n.Add(nd)
if err != nil {
log.Info("AddRecursive Error: %s\n", err)
return err
}
for _, link := range nd.Links {
if link.Node != nil {
err := n.AddRecursive(link.Node)
if err != nil {
return err
}
}
}
return nil
}
// Get retrieves a node from the dagService, fetching the block in the BlockService
func (n *dagService) Get(ctx context.Context, k key.Key) (*Node, error) {
if n == nil {
@ -101,24 +79,14 @@ func (n *dagService) Get(ctx context.Context, k key.Key) (*Node, error) {
if err == bserv.ErrNotFound {
return nil, ErrNotFound
}
return nil, err
return nil, fmt.Errorf("Failed to get block for %s: %v", k.B58String(), err)
}
return Decoded(b.Data)
}
// Remove deletes the given node and all of its children from the BlockService
func (n *dagService) RemoveRecursive(nd *Node) error {
for _, l := range nd.Links {
if l.Node != nil {
n.RemoveRecursive(l.Node)
}
}
k, err := nd.Key()
res, err := DecodeProtobuf(b.Data)
if err != nil {
return err
return nil, fmt.Errorf("Failed to decode Protocol Buffers: %v", err)
}
return n.Blocks.DeleteBlock(k)
return res, nil
}
func (n *dagService) Remove(nd *Node) error {
@ -167,7 +135,7 @@ func (ds *dagService) GetMany(ctx context.Context, keys []key.Key) <-chan *NodeO
}
return
}
nd, err := Decoded(b.Data)
nd, err := DecodeProtobuf(b.Data)
if err != nil {
out <- &NodeOption{Err: err}
return
@ -313,7 +281,7 @@ type Batch struct {
}
func (t *Batch) Add(nd *Node) (key.Key, error) {
d, err := nd.Encoded(false)
d, err := nd.EncodeProtobuf(false)
if err != nil {
return "", err
}

47
merkledag/merkledag_test.go
View File

@ -64,7 +64,7 @@ func TestNode(t *testing.T) {
fmt.Println("-", l.Name, l.Size, l.Hash)
}
e, err := n.Encoded(false)
e, err := n.EncodeProtobuf(false)
if err != nil {
t.Error(err)
} else {
@ -96,9 +96,9 @@ func TestNode(t *testing.T) {
}
func SubtestNodeStat(t *testing.T, n *Node) {
enc, err := n.Encoded(true)
enc, err := n.EncodeProtobuf(true)
if err != nil {
t.Error("n.Encoded(true) failed")
t.Error("n.EncodeProtobuf(true) failed")
return
}
@ -181,7 +181,7 @@ func runBatchFetchTest(t *testing.T, read io.Reader) {
t.Fatal(err)
}
err = dagservs[0].AddRecursive(root)
_, err = dagservs[0].Add(root)
if err != nil {
t.Fatal(err)
}
@ -232,45 +232,6 @@ func runBatchFetchTest(t *testing.T, read io.Reader) {
}
}
}
func TestRecursiveAdd(t *testing.T) {
a := &Node{Data: []byte("A")}
b := &Node{Data: []byte("B")}
c := &Node{Data: []byte("C")}
d := &Node{Data: []byte("D")}
e := &Node{Data: []byte("E")}
err := a.AddNodeLink("blah", b)
if err != nil {
t.Fatal(err)
}
err = b.AddNodeLink("foo", c)
if err != nil {
t.Fatal(err)
}
err = b.AddNodeLink("bar", d)
if err != nil {
t.Fatal(err)
}
err = d.AddNodeLink("baz", e)
if err != nil {
t.Fatal(err)
}
dsp := getDagservAndPinner(t)
err = dsp.ds.AddRecursive(a)
if err != nil {
t.Fatal(err)
}
assertCanGet(t, dsp.ds, a)
assertCanGet(t, dsp.ds, b)
assertCanGet(t, dsp.ds, c)
assertCanGet(t, dsp.ds, d)
assertCanGet(t, dsp.ds, e)
}
func assertCanGet(t *testing.T, ds DAGService, n *Node) {
k, err := n.Key()

20
merkledag/node.go
View File

@ -5,8 +5,8 @@ import (
"gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context"
mh "gx/ipfs/QmYf7ng2hG5XBtJA3tN34DQ2GUN5HNksEw1rLDkmr6vGku/go-multihash"
key "github.com/ipfs/go-ipfs/blocks/key"
mh "gx/ipfs/QmYf7ng2hG5XBtJA3tN34DQ2GUN5HNksEw1rLDkmr6vGku/go-multihash"
)
var ErrLinkNotFound = fmt.Errorf("no link by that name")
@ -48,9 +48,6 @@ type Link struct {
// multihash of the target object
Hash mh.Multihash
// a ptr to the actual node for graph manipulation
Node *Node
}
type LinkSlice []*Link
@ -78,10 +75,6 @@ func MakeLink(n *Node) (*Link, error) {
// GetNode returns the MDAG Node that this link points to
func (l *Link) GetNode(ctx context.Context, serv DAGService) (*Node, error) {
if l.Node != nil {
return l.Node, nil
}
return serv.Get(ctx, key.Key(l.Hash))
}
@ -92,7 +85,6 @@ func (n *Node) AddNodeLink(name string, that *Node) error {
lnk, err := MakeLink(that)
lnk.Name = name
lnk.Node = that
if err != nil {
return err
}
@ -122,7 +114,6 @@ func (n *Node) AddRawLink(name string, l *Link) error {
Name: name,
Size: l.Size,
Hash: l.Hash,
Node: l.Node,
})
return nil
@ -158,7 +149,6 @@ func (n *Node) GetNodeLink(name string) (*Link, error) {
Name: l.Name,
Size: l.Size,
Hash: l.Hash,
Node: l.Node,
}, nil
}
}
@ -203,7 +193,7 @@ func (n *Node) UpdateNodeLink(name string, that *Node) (*Node, error) {
// Size returns the total size of the data addressed by node,
// including the total sizes of references.
func (n *Node) Size() (uint64, error) {
b, err := n.Encoded(false)
b, err := n.EncodeProtobuf(false)
if err != nil {
return 0, err
}
@ -217,7 +207,7 @@ func (n *Node) Size() (uint64, error) {
// Stat returns statistics on the node.
func (n *Node) Stat() (*NodeStat, error) {
enc, err := n.Encoded(false)
enc, err := n.EncodeProtobuf(false)
if err != nil {
return nil, err
}
@ -244,8 +234,8 @@ func (n *Node) Stat() (*NodeStat, error) {
// Multihash hashes the encoded data of this node.
func (n *Node) Multihash() (mh.Multihash, error) {
// Note: Encoded generates the hash and puts it in n.cached.
_, err := n.Encoded(false)
// Note: EncodeProtobuf generates the hash and puts it in n.cached.
_, err := n.EncodeProtobuf(false)
if err != nil {
return nil, err
}

140
merkledag/traverse/traverse_test.go
View File

@ -6,12 +6,14 @@ import (
"testing"
mdag "github.com/ipfs/go-ipfs/merkledag"
mdagtest "github.com/ipfs/go-ipfs/merkledag/test"
)
func TestDFSPreNoSkip(t *testing.T) {
opts := Options{Order: DFSPre}
ds := mdagtest.Mock()
opts := Options{Order: DFSPre, DAG: ds}
testWalkOutputs(t, newFan(t), opts, []byte(`
testWalkOutputs(t, newFan(t, ds), opts, []byte(`
0 /a
1 /a/aa
1 /a/ab
@ -19,7 +21,7 @@ func TestDFSPreNoSkip(t *testing.T) {
1 /a/ad
`))
testWalkOutputs(t, newLinkedList(t), opts, []byte(`
testWalkOutputs(t, newLinkedList(t, ds), opts, []byte(`
0 /a
1 /a/aa
2 /a/aa/aaa
@ -27,7 +29,7 @@ func TestDFSPreNoSkip(t *testing.T) {
4 /a/aa/aaa/aaaa/aaaaa
`))
testWalkOutputs(t, newBinaryTree(t), opts, []byte(`
testWalkOutputs(t, newBinaryTree(t, ds), opts, []byte(`
0 /a
1 /a/aa
2 /a/aa/aaa
@ -37,7 +39,7 @@ func TestDFSPreNoSkip(t *testing.T) {
2 /a/ab/abb
`))
testWalkOutputs(t, newBinaryDAG(t), opts, []byte(`
testWalkOutputs(t, newBinaryDAG(t, ds), opts, []byte(`
0 /a
1 /a/aa
2 /a/aa/aaa
@ -73,9 +75,10 @@ func TestDFSPreNoSkip(t *testing.T) {
}
func TestDFSPreSkip(t *testing.T) {
opts := Options{Order: DFSPre, SkipDuplicates: true}
ds := mdagtest.Mock()
opts := Options{Order: DFSPre, SkipDuplicates: true, DAG: ds}
testWalkOutputs(t, newFan(t), opts, []byte(`
testWalkOutputs(t, newFan(t, ds), opts, []byte(`
0 /a
1 /a/aa
1 /a/ab
@ -83,7 +86,7 @@ func TestDFSPreSkip(t *testing.T) {
1 /a/ad
`))
testWalkOutputs(t, newLinkedList(t), opts, []byte(`
testWalkOutputs(t, newLinkedList(t, ds), opts, []byte(`
0 /a
1 /a/aa
2 /a/aa/aaa
@ -91,7 +94,7 @@ func TestDFSPreSkip(t *testing.T) {
4 /a/aa/aaa/aaaa/aaaaa
`))
testWalkOutputs(t, newBinaryTree(t), opts, []byte(`
testWalkOutputs(t, newBinaryTree(t, ds), opts, []byte(`
0 /a
1 /a/aa
2 /a/aa/aaa
@ -101,7 +104,7 @@ func TestDFSPreSkip(t *testing.T) {
2 /a/ab/abb
`))
testWalkOutputs(t, newBinaryDAG(t), opts, []byte(`
testWalkOutputs(t, newBinaryDAG(t, ds), opts, []byte(`
0 /a
1 /a/aa
2 /a/aa/aaa
@ -111,9 +114,10 @@ func TestDFSPreSkip(t *testing.T) {
}
func TestDFSPostNoSkip(t *testing.T) {
opts := Options{Order: DFSPost}
ds := mdagtest.Mock()
opts := Options{Order: DFSPost, DAG: ds}
testWalkOutputs(t, newFan(t), opts, []byte(`
testWalkOutputs(t, newFan(t, ds), opts, []byte(`
1 /a/aa
1 /a/ab
1 /a/ac
@ -121,7 +125,7 @@ func TestDFSPostNoSkip(t *testing.T) {
0 /a
`))
testWalkOutputs(t, newLinkedList(t), opts, []byte(`
testWalkOutputs(t, newLinkedList(t, ds), opts, []byte(`
4 /a/aa/aaa/aaaa/aaaaa
3 /a/aa/aaa/aaaa
2 /a/aa/aaa
@ -129,7 +133,7 @@ func TestDFSPostNoSkip(t *testing.T) {
0 /a
`))
testWalkOutputs(t, newBinaryTree(t), opts, []byte(`
testWalkOutputs(t, newBinaryTree(t, ds), opts, []byte(`
2 /a/aa/aaa
2 /a/aa/aab
1 /a/aa
@ -139,7 +143,7 @@ func TestDFSPostNoSkip(t *testing.T) {
0 /a
`))
testWalkOutputs(t, newBinaryDAG(t), opts, []byte(`
testWalkOutputs(t, newBinaryDAG(t, ds), opts, []byte(`
4 /a/aa/aaa/aaaa/aaaaa
4 /a/aa/aaa/aaaa/aaaaa
3 /a/aa/aaa/aaaa
@ -175,9 +179,10 @@ func TestDFSPostNoSkip(t *testing.T) {
}
func TestDFSPostSkip(t *testing.T) {
opts := Options{Order: DFSPost, SkipDuplicates: true}
ds := mdagtest.Mock()
opts := Options{Order: DFSPost, SkipDuplicates: true, DAG: ds}
testWalkOutputs(t, newFan(t), opts, []byte(`
testWalkOutputs(t, newFan(t, ds), opts, []byte(`
1 /a/aa
1 /a/ab
1 /a/ac
@ -185,7 +190,7 @@ func TestDFSPostSkip(t *testing.T) {
0 /a
`))
testWalkOutputs(t, newLinkedList(t), opts, []byte(`
testWalkOutputs(t, newLinkedList(t, ds), opts, []byte(`
4 /a/aa/aaa/aaaa/aaaaa
3 /a/aa/aaa/aaaa
2 /a/aa/aaa
@ -193,7 +198,7 @@ func TestDFSPostSkip(t *testing.T) {
0 /a
`))
testWalkOutputs(t, newBinaryTree(t), opts, []byte(`
testWalkOutputs(t, newBinaryTree(t, ds), opts, []byte(`
2 /a/aa/aaa
2 /a/aa/aab
1 /a/aa
@ -203,7 +208,7 @@ func TestDFSPostSkip(t *testing.T) {
0 /a
`))
testWalkOutputs(t, newBinaryDAG(t), opts, []byte(`
testWalkOutputs(t, newBinaryDAG(t, ds), opts, []byte(`
4 /a/aa/aaa/aaaa/aaaaa
3 /a/aa/aaa/aaaa
2 /a/aa/aaa
@ -213,9 +218,10 @@ func TestDFSPostSkip(t *testing.T) {
}
func TestBFSNoSkip(t *testing.T) {
opts := Options{Order: BFS}
ds := mdagtest.Mock()
opts := Options{Order: BFS, DAG: ds}
testWalkOutputs(t, newFan(t), opts, []byte(`
testWalkOutputs(t, newFan(t, ds), opts, []byte(`
0 /a
1 /a/aa
1 /a/ab
@ -223,7 +229,7 @@ func TestBFSNoSkip(t *testing.T) {
1 /a/ad
`))
testWalkOutputs(t, newLinkedList(t), opts, []byte(`
testWalkOutputs(t, newLinkedList(t, ds), opts, []byte(`
0 /a
1 /a/aa
2 /a/aa/aaa
@ -231,7 +237,7 @@ func TestBFSNoSkip(t *testing.T) {
4 /a/aa/aaa/aaaa/aaaaa
`))
testWalkOutputs(t, newBinaryTree(t), opts, []byte(`
testWalkOutputs(t, newBinaryTree(t, ds), opts, []byte(`
0 /a
1 /a/aa
1 /a/ab
@ -241,7 +247,7 @@ func TestBFSNoSkip(t *testing.T) {
2 /a/ab/abb
`))
testWalkOutputs(t, newBinaryDAG(t), opts, []byte(`
testWalkOutputs(t, newBinaryDAG(t, ds), opts, []byte(`
0 /a
1 /a/aa
1 /a/aa
@ -277,9 +283,10 @@ func TestBFSNoSkip(t *testing.T) {
}
func TestBFSSkip(t *testing.T) {
opts := Options{Order: BFS, SkipDuplicates: true}
ds := mdagtest.Mock()
opts := Options{Order: BFS, SkipDuplicates: true, DAG: ds}
testWalkOutputs(t, newFan(t), opts, []byte(`
testWalkOutputs(t, newFan(t, ds), opts, []byte(`
0 /a
1 /a/aa
1 /a/ab
@ -287,7 +294,7 @@ func TestBFSSkip(t *testing.T) {
1 /a/ad
`))
testWalkOutputs(t, newLinkedList(t), opts, []byte(`
testWalkOutputs(t, newLinkedList(t, ds), opts, []byte(`
0 /a
1 /a/aa
2 /a/aa/aaa
@ -295,7 +302,7 @@ func TestBFSSkip(t *testing.T) {
4 /a/aa/aaa/aaaa/aaaaa
`))
testWalkOutputs(t, newBinaryTree(t), opts, []byte(`
testWalkOutputs(t, newBinaryTree(t, ds), opts, []byte(`
0 /a
1 /a/aa
1 /a/ab
@ -305,7 +312,7 @@ func TestBFSSkip(t *testing.T) {
2 /a/ab/abb
`))
testWalkOutputs(t, newBinaryDAG(t), opts, []byte(`
testWalkOutputs(t, newBinaryDAG(t, ds), opts, []byte(`
0 /a
1 /a/aa
2 /a/aa/aaa
@ -341,57 +348,68 @@ func testWalkOutputs(t *testing.T, root *mdag.Node, opts Options, expect []byte)
}
}
func newFan(t *testing.T) *mdag.Node {
func newFan(t *testing.T, ds mdag.DAGService) *mdag.Node {
a := &mdag.Node{Data: []byte("/a")}
addChild(t, a, "aa")
addChild(t, a, "ab")
addChild(t, a, "ac")
addChild(t, a, "ad")
addLink(t, ds, a, child(t, ds, a, "aa"))
addLink(t, ds, a, child(t, ds, a, "ab"))
addLink(t, ds, a, child(t, ds, a, "ac"))
addLink(t, ds, a, child(t, ds, a, "ad"))
return a
}
func newLinkedList(t *testing.T) *mdag.Node {
func newLinkedList(t *testing.T, ds mdag.DAGService) *mdag.Node {
a := &mdag.Node{Data: []byte("/a")}
aa := addChild(t, a, "aa")
aaa := addChild(t, aa, "aaa")
aaaa := addChild(t, aaa, "aaaa")
addChild(t, aaaa, "aaaaa")
aa := child(t, ds, a, "aa")
aaa := child(t, ds, aa, "aaa")
aaaa := child(t, ds, aaa, "aaaa")
aaaaa := child(t, ds, aaaa, "aaaaa")
addLink(t, ds, aaaa, aaaaa)
addLink(t, ds, aaa, aaaa)
addLink(t, ds, aa, aaa)
addLink(t, ds, a, aa)
return a
}
func newBinaryTree(t *testing.T) *mdag.Node {
func newBinaryTree(t *testing.T, ds mdag.DAGService) *mdag.Node {
a := &mdag.Node{Data: []byte("/a")}
aa := addChild(t, a, "aa")
ab := addChild(t, a, "ab")
addChild(t, aa, "aaa")
addChild(t, aa, "aab")
addChild(t, ab, "aba")
addChild(t, ab, "abb")
aa := child(t, ds, a, "aa")
ab := child(t, ds, a, "ab")
addLink(t, ds, aa, child(t, ds, aa, "aaa"))
addLink(t, ds, aa, child(t, ds, aa, "aab"))
addLink(t, ds, ab, child(t, ds, ab, "aba"))
addLink(t, ds, ab, child(t, ds, ab, "abb"))
addLink(t, ds, a, aa)
addLink(t, ds, a, ab)
return a
}
func newBinaryDAG(t *testing.T) *mdag.Node {
func newBinaryDAG(t *testing.T, ds mdag.DAGService) *mdag.Node {
a := &mdag.Node{Data: []byte("/a")}
aa := addChild(t, a, "aa")
aaa := addChild(t, aa, "aaa")
aaaa := addChild(t, aaa, "aaaa")
aaaaa := addChild(t, aaaa, "aaaaa")
addLink(t, a, aa)
addLink(t, aa, aaa)
addLink(t, aaa, aaaa)
addLink(t, aaaa, aaaaa)
aa := child(t, ds, a, "aa")
aaa := child(t, ds, aa, "aaa")
aaaa := child(t, ds, aaa, "aaaa")
aaaaa := child(t, ds, aaaa, "aaaaa")
addLink(t, ds, aaaa, aaaaa)
addLink(t, ds, aaaa, aaaaa)
addLink(t, ds, aaa, aaaa)
addLink(t, ds, aaa, aaaa)
addLink(t, ds, aa, aaa)
addLink(t, ds, aa, aaa)
addLink(t, ds, a, aa)
addLink(t, ds, a, aa)
return a
}
func addLink(t *testing.T, a, b *mdag.Node) {
func addLink(t *testing.T, ds mdag.DAGService, a, b *mdag.Node) {
to := string(a.Data) + "2" + string(b.Data)
if _, err := ds.Add(b); err != nil {
t.Error(err)
}
if err := a.AddNodeLink(to, b); err != nil {
t.Error(err)
}
}
func addChild(t *testing.T, a *mdag.Node, name string) *mdag.Node {
c := &mdag.Node{Data: []byte(string(a.Data) + "/" + name)}
addLink(t, a, c)
return c
func child(t *testing.T, ds mdag.DAGService, a *mdag.Node, name string) *mdag.Node {
return &mdag.Node{Data: []byte(string(a.Data) + "/" + name)}
}

35
path/resolver.go
View File

@ -111,37 +111,20 @@ func (s *Resolver) ResolveLinks(ctx context.Context, ndd *merkledag.Node, names
// for each of the path components
for _, name := range names {
var next key.Key
var nlink *merkledag.Link
// for each of the links in nd, the current object
for _, link := range nd.Links {
if link.Name == name {
next = key.Key(link.Hash)
nlink = link
break
}
}
var cancel context.CancelFunc
ctx, cancel = context.WithTimeout(ctx, time.Minute)
defer cancel()
if next == "" {
nextnode, err := nd.GetLinkedNode(ctx, s.DAG, name)
if err == merkledag.ErrLinkNotFound {
n, _ := nd.Multihash()
return result, ErrNoLink{Name: name, Node: n}
} else if err != nil {
return append(result, nextnode), err
}
if nlink.Node == nil {
// fetch object for link and assign to nd
ctx, cancel := context.WithTimeout(ctx, time.Minute)
defer cancel()
var err error
nd, err = s.DAG.Get(ctx, next)
if err != nil {
return append(result, nd), err
}
nlink.Node = nd
} else {
nd = nlink.Node
}
result = append(result, nlink.Node)
nd = nextnode
result = append(result, nextnode)
}
return result, nil
}

8
path/resolver_test.go
View File

@ -39,9 +39,11 @@ func TestRecurivePathResolution(t *testing.T) {
t.Fatal(err)
}
err = dagService.AddRecursive(a)
if err != nil {
t.Fatal(err)
for _, n := range []*merkledag.Node{a, b, c} {
_, err = dagService.Add(n)
if err != nil {
t.Fatal(err)
}
}
aKey, err := a.Key()

6
pin/pin_test.go
View File

@ -104,7 +104,11 @@ func TestPinnerBasic(t *testing.T) {
d.AddNodeLink("e", e)
// Must be in dagserv for unpin to work
err = dserv.AddRecursive(d)
_, err = dserv.Add(e)
if err != nil {
t.Fatal(err)
}
_, err = dserv.Add(d)
if err != nil {
t.Fatal(err)
}

7
pin/set.go
View File

@ -11,10 +11,10 @@ import (
"sort"
"unsafe"
"gx/ipfs/QmZ4Qi3GaRbjcx28Sme5eMH7RQjGkt8wHxt2a65oLaeFEV/gogo-protobuf/proto"
"github.com/ipfs/go-ipfs/blocks/key"
"github.com/ipfs/go-ipfs/merkledag"
"github.com/ipfs/go-ipfs/pin/internal/pb"
"gx/ipfs/QmZ4Qi3GaRbjcx28Sme5eMH7RQjGkt8wHxt2a65oLaeFEV/gogo-protobuf/proto"
"gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context"
)
@ -172,7 +172,6 @@ func storeItems(ctx context.Context, dag merkledag.DAGService, estimatedLen uint
Name: "",
Hash: childKey.ToMultihash(),
Size: size,
Node: child,
}
n.Links[int(h%defaultFanout)] = l
}
@ -272,12 +271,12 @@ func loadSet(ctx context.Context, dag merkledag.DAGService, root *merkledag.Node
func loadMultiset(ctx context.Context, dag merkledag.DAGService, root *merkledag.Node, name string, internalKeys keyObserver) (map[key.Key]uint64, error) {
l, err := root.GetNodeLink(name)
if err != nil {
return nil, err
return nil, fmt.Errorf("Failed to get link %s: %v", name, err)
}
internalKeys(key.Key(l.Hash))
n, err := l.GetNode(ctx, dag)
if err != nil {
return nil, err
return nil, fmt.Errorf("Failed to get node from link %s: %v", name, err)
}
refcounts := make(map[key.Key]uint64)

21
pin/set_test.go
View File

@ -11,6 +11,8 @@ import (
"github.com/ipfs/go-ipfs/blockservice"
"github.com/ipfs/go-ipfs/exchange/offline"
"github.com/ipfs/go-ipfs/merkledag"
mh "gx/ipfs/QmYf7ng2hG5XBtJA3tN34DQ2GUN5HNksEw1rLDkmr6vGku/go-multihash"
u "gx/ipfs/QmZNVWh8LLjAavuQ2JXuFmuYH3C11xo988vSgp7UQrTRj1/go-ipfs-util"
"gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context"
)
@ -31,6 +33,14 @@ func TestMultisetRoundtrip(t *testing.T) {
dag := merkledag.NewDAGService(bserv)
fn := func(m map[key.Key]uint16) bool {
// Convert invalid multihash from input to valid ones
for k, v := range m {
if _, err := mh.Cast([]byte(k)); err != nil {
delete(m, k)
m[key.Key(u.Hash([]byte(k)))] = v
}
}
// Generate a smaller range for refcounts than full uint64, as
// otherwise this just becomes overly cpu heavy, splitting it
// out into too many items. That means we need to convert to
@ -43,6 +53,17 @@ func TestMultisetRoundtrip(t *testing.T) {
if err != nil {
t.Fatalf("storing multiset: %v", err)
}
// Check that the node n is in the DAG
k, err := n.Key()
if err != nil {
t.Fatalf("Could not get key: %v", err)
}
_, err = dag.Get(ctx, k)
if err != nil {
t.Fatalf("Could not get node: %v", err)
}
root := &merkledag.Node{}
const linkName = "dummylink"
if err := root.AddNodeLink(linkName, n); err != nil {

4
unixfs/mod/dagmodifier.go
View File

@ -258,7 +258,7 @@ func (dm *DagModifier) modifyDag(node *mdag.Node, offset uint64, data io.Reader)
node.Links[i].Hash = mh.Multihash(k)
// Recache serialized node
_, err = node.Encoded(true)
_, err = node.EncodeProtobuf(true)
if err != nil {
return "", false, err
}
@ -489,7 +489,7 @@ func dagTruncate(ctx context.Context, nd *mdag.Node, size uint64, ds mdag.DAGSer
nd.Data = d
// invalidate cache and recompute serialized data
_, err = nd.Encoded(true)
_, err = nd.EncodeProtobuf(true)
if err != nil {
return nil, err
}