mirror of
https://github.com/ipfs/kubo.git
synced 2025-08-06 19:44:01 +08:00
454 lines
9.0 KiB
Go
454 lines
9.0 KiB
Go
package merkledag_test
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"io/ioutil"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
|
|
bserv "github.com/ipfs/go-ipfs/blockservice"
|
|
bstest "github.com/ipfs/go-ipfs/blockservice/test"
|
|
offline "github.com/ipfs/go-ipfs/exchange/offline"
|
|
imp "github.com/ipfs/go-ipfs/importer"
|
|
chunk "github.com/ipfs/go-ipfs/importer/chunk"
|
|
. "github.com/ipfs/go-ipfs/merkledag"
|
|
mdpb "github.com/ipfs/go-ipfs/merkledag/pb"
|
|
dstest "github.com/ipfs/go-ipfs/merkledag/test"
|
|
uio "github.com/ipfs/go-ipfs/unixfs/io"
|
|
|
|
cid "gx/ipfs/QmXUuRadqDq5BuFWzVU6VuKaSjTcNm1gNCtLvvP1TJCW4z/go-cid"
|
|
key "gx/ipfs/QmYEoKZXHoAToWfhGF3vryhMn3WWhE1o2MasQ8uzY5iDi9/go-key"
|
|
node "gx/ipfs/QmZx42H5khbVQhV5odp66TApShV4XCujYazcvYduZ4TroB/go-ipld-node"
|
|
u "gx/ipfs/Qmb912gdngC1UWwTkhuW8knyRbcWeu5kqkxBpveLmW8bSr/go-ipfs-util"
|
|
)
|
|
|
|
func TestNode(t *testing.T) {
|
|
|
|
n1 := NodeWithData([]byte("beep"))
|
|
n2 := NodeWithData([]byte("boop"))
|
|
n3 := NodeWithData([]byte("beep boop"))
|
|
if err := n3.AddNodeLink("beep-link", n1); err != nil {
|
|
t.Error(err)
|
|
}
|
|
if err := n3.AddNodeLink("boop-link", n2); err != nil {
|
|
t.Error(err)
|
|
}
|
|
|
|
printn := func(name string, n *ProtoNode) {
|
|
fmt.Println(">", name)
|
|
fmt.Println("data:", string(n.Data()))
|
|
|
|
fmt.Println("links:")
|
|
for _, l := range n.Links() {
|
|
fmt.Println("-", l.Name, l.Size, l.Cid)
|
|
}
|
|
|
|
e, err := n.EncodeProtobuf(false)
|
|
if err != nil {
|
|
t.Error(err)
|
|
} else {
|
|
fmt.Println("encoded:", e)
|
|
}
|
|
|
|
h := n.Multihash()
|
|
k := n.Key()
|
|
if k != key.Key(h) {
|
|
t.Error("Key is not equivalent to multihash")
|
|
} else {
|
|
fmt.Println("key: ", k)
|
|
}
|
|
|
|
SubtestNodeStat(t, n)
|
|
}
|
|
|
|
printn("beep", n1)
|
|
printn("boop", n2)
|
|
printn("beep boop", n3)
|
|
}
|
|
|
|
func SubtestNodeStat(t *testing.T, n *ProtoNode) {
|
|
enc, err := n.EncodeProtobuf(true)
|
|
if err != nil {
|
|
t.Error("n.EncodeProtobuf(true) failed")
|
|
return
|
|
}
|
|
|
|
cumSize, err := n.Size()
|
|
if err != nil {
|
|
t.Error("n.Size() failed")
|
|
return
|
|
}
|
|
|
|
k := n.Key()
|
|
|
|
expected := node.NodeStat{
|
|
NumLinks: len(n.Links()),
|
|
BlockSize: len(enc),
|
|
LinksSize: len(enc) - len(n.Data()), // includes framing.
|
|
DataSize: len(n.Data()),
|
|
CumulativeSize: int(cumSize),
|
|
Hash: k.B58String(),
|
|
}
|
|
|
|
actual, err := n.Stat()
|
|
if err != nil {
|
|
t.Error("n.Stat() failed")
|
|
return
|
|
}
|
|
|
|
if expected != *actual {
|
|
t.Errorf("n.Stat incorrect.\nexpect: %s\nactual: %s", expected, actual)
|
|
} else {
|
|
fmt.Printf("n.Stat correct: %s\n", actual)
|
|
}
|
|
}
|
|
|
|
type devZero struct{}
|
|
|
|
func (_ devZero) Read(b []byte) (int, error) {
|
|
for i := range b {
|
|
b[i] = 0
|
|
}
|
|
return len(b), nil
|
|
}
|
|
|
|
func TestBatchFetch(t *testing.T) {
|
|
read := io.LimitReader(u.NewTimeSeededRand(), 1024*32)
|
|
runBatchFetchTest(t, read)
|
|
}
|
|
|
|
func TestBatchFetchDupBlock(t *testing.T) {
|
|
read := io.LimitReader(devZero{}, 1024*32)
|
|
runBatchFetchTest(t, read)
|
|
}
|
|
|
|
func runBatchFetchTest(t *testing.T, read io.Reader) {
|
|
ctx := context.Background()
|
|
var dagservs []DAGService
|
|
for _, bsi := range bstest.Mocks(5) {
|
|
dagservs = append(dagservs, NewDAGService(bsi))
|
|
}
|
|
|
|
spl := chunk.NewSizeSplitter(read, 512)
|
|
|
|
root, err := imp.BuildDagFromReader(dagservs[0], spl)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
t.Log("finished setup.")
|
|
|
|
dagr, err := uio.NewDagReader(ctx, root, dagservs[0])
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
expected, err := ioutil.ReadAll(dagr)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
_, err = dagservs[0].Add(root)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
t.Log("Added file to first node.")
|
|
|
|
c := root.Cid()
|
|
|
|
wg := sync.WaitGroup{}
|
|
errs := make(chan error)
|
|
|
|
for i := 1; i < len(dagservs); i++ {
|
|
wg.Add(1)
|
|
go func(i int) {
|
|
defer wg.Done()
|
|
first, err := dagservs[i].Get(ctx, c)
|
|
if err != nil {
|
|
errs <- err
|
|
}
|
|
fmt.Println("Got first node back.")
|
|
|
|
firstpb, ok := first.(*ProtoNode)
|
|
if !ok {
|
|
errs <- ErrNotProtobuf
|
|
}
|
|
|
|
read, err := uio.NewDagReader(ctx, firstpb, dagservs[i])
|
|
if err != nil {
|
|
errs <- err
|
|
}
|
|
datagot, err := ioutil.ReadAll(read)
|
|
if err != nil {
|
|
errs <- err
|
|
}
|
|
|
|
if !bytes.Equal(datagot, expected) {
|
|
errs <- errors.New("Got bad data back!")
|
|
}
|
|
}(i)
|
|
}
|
|
|
|
go func() {
|
|
wg.Wait()
|
|
close(errs)
|
|
}()
|
|
|
|
for err := range errs {
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
}
|
|
|
|
func assertCanGet(t *testing.T, ds DAGService, n node.Node) {
|
|
if _, err := ds.Get(context.Background(), n.Cid()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func TestCantGet(t *testing.T) {
|
|
ds := dstest.Mock()
|
|
a := NodeWithData([]byte("A"))
|
|
|
|
c := a.Cid()
|
|
_, err := ds.Get(context.Background(), c)
|
|
if !strings.Contains(err.Error(), "not found") {
|
|
t.Fatal("expected err not found, got: ", err)
|
|
}
|
|
}
|
|
|
|
func TestFetchGraph(t *testing.T) {
|
|
var dservs []DAGService
|
|
bsis := bstest.Mocks(2)
|
|
for _, bsi := range bsis {
|
|
dservs = append(dservs, NewDAGService(bsi))
|
|
}
|
|
|
|
read := io.LimitReader(u.NewTimeSeededRand(), 1024*32)
|
|
root, err := imp.BuildDagFromReader(dservs[0], chunk.NewSizeSplitter(read, 512))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
err = FetchGraph(context.TODO(), root.Cid(), dservs[1])
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
// create an offline dagstore and ensure all blocks were fetched
|
|
bs := bserv.New(bsis[1].Blockstore(), offline.Exchange(bsis[1].Blockstore()))
|
|
|
|
offline_ds := NewDAGService(bs)
|
|
|
|
err = EnumerateChildren(context.Background(), offline_ds, root.Cid(), func(_ *cid.Cid) bool { return true }, false)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func TestEnumerateChildren(t *testing.T) {
|
|
bsi := bstest.Mocks(1)
|
|
ds := NewDAGService(bsi[0])
|
|
|
|
read := io.LimitReader(u.NewTimeSeededRand(), 1024*1024)
|
|
root, err := imp.BuildDagFromReader(ds, chunk.NewSizeSplitter(read, 512))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
set := cid.NewSet()
|
|
err = EnumerateChildren(context.Background(), ds, root.Cid(), set.Visit, false)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
var traverse func(n node.Node)
|
|
traverse = func(n node.Node) {
|
|
// traverse dag and check
|
|
for _, lnk := range n.Links() {
|
|
c := lnk.Cid
|
|
if !set.Has(c) {
|
|
t.Fatal("missing key in set! ", lnk.Cid.String())
|
|
}
|
|
child, err := ds.Get(context.Background(), c)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
traverse(child)
|
|
}
|
|
}
|
|
|
|
traverse(root)
|
|
}
|
|
|
|
func TestFetchFailure(t *testing.T) {
|
|
ds := dstest.Mock()
|
|
ds_bad := dstest.Mock()
|
|
|
|
top := new(ProtoNode)
|
|
for i := 0; i < 10; i++ {
|
|
nd := NodeWithData([]byte{byte('a' + i)})
|
|
_, err := ds.Add(nd)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
err = top.AddNodeLinkClean(fmt.Sprintf("AA%d", i), nd)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
for i := 0; i < 10; i++ {
|
|
nd := NodeWithData([]byte{'f', 'a' + byte(i)})
|
|
_, err := ds_bad.Add(nd)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
err = top.AddNodeLinkClean(fmt.Sprintf("BB%d", i), nd)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
getters := GetDAG(context.Background(), ds, top)
|
|
for i, getter := range getters {
|
|
_, err := getter.Get(context.Background())
|
|
if err != nil && i < 10 {
|
|
t.Fatal(err)
|
|
}
|
|
if err == nil && i >= 10 {
|
|
t.Fatal("should have failed request")
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestUnmarshalFailure(t *testing.T) {
|
|
badData := []byte("hello world")
|
|
|
|
_, err := DecodeProtobuf(badData)
|
|
if err == nil {
|
|
t.Fatal("shouldnt succeed to parse this")
|
|
}
|
|
|
|
// now with a bad link
|
|
pbn := &mdpb.PBNode{Links: []*mdpb.PBLink{{Hash: []byte("not a multihash")}}}
|
|
badlink, err := pbn.Marshal()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
_, err = DecodeProtobuf(badlink)
|
|
if err == nil {
|
|
t.Fatal("should have failed to parse node with bad link")
|
|
}
|
|
|
|
n := &ProtoNode{}
|
|
n.Marshal()
|
|
}
|
|
|
|
func TestBasicAddGet(t *testing.T) {
|
|
ds := dstest.Mock()
|
|
nd := new(ProtoNode)
|
|
|
|
c, err := ds.Add(nd)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
out, err := ds.Get(context.Background(), c)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
if !nd.Cid().Equals(out.Cid()) {
|
|
t.Fatal("output didnt match input")
|
|
}
|
|
}
|
|
|
|
func TestGetRawNodes(t *testing.T) {
|
|
rn := NewRawNode([]byte("test"))
|
|
|
|
ds := dstest.Mock()
|
|
|
|
c, err := ds.Add(rn)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
if !c.Equals(rn.Cid()) {
|
|
t.Fatal("output cids didnt match")
|
|
}
|
|
|
|
out, err := ds.Get(context.TODO(), c)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
if !bytes.Equal(out.RawData(), []byte("test")) {
|
|
t.Fatal("raw block should match input data")
|
|
}
|
|
|
|
if out.Links() != nil {
|
|
t.Fatal("raw blocks shouldnt have links")
|
|
}
|
|
|
|
if out.Tree() != nil {
|
|
t.Fatal("tree should return no paths in a raw block")
|
|
}
|
|
|
|
size, err := out.Size()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if size != 4 {
|
|
t.Fatal("expected size to be 4")
|
|
}
|
|
|
|
ns, err := out.Stat()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
if ns.DataSize != 4 {
|
|
t.Fatal("expected size to be 4, got: ", ns.DataSize)
|
|
}
|
|
|
|
_, _, err = out.Resolve([]string{"foo"})
|
|
if err != ErrLinkNotFound {
|
|
t.Fatal("shouldnt find links under raw blocks")
|
|
}
|
|
}
|
|
|
|
func TestProtoNodeResolve(t *testing.T) {
|
|
|
|
nd := new(ProtoNode)
|
|
nd.SetLinks([]*node.Link{{Name: "foo"}})
|
|
|
|
lnk, left, err := nd.Resolve([]string{"foo", "bar"})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
if len(left) != 1 || left[0] != "bar" {
|
|
t.Fatal("expected the single path element 'bar' to remain")
|
|
}
|
|
|
|
if lnk.Name != "foo" {
|
|
t.Fatal("how did we get anything else?")
|
|
}
|
|
|
|
tvals := nd.Tree()
|
|
if len(tvals) != 1 || tvals[0] != "foo" {
|
|
t.Fatal("expected tree to return []{\"foo\"}")
|
|
}
|
|
}
|