From 856c87bdc093b6a6642b262fd4c1321bb94e9c62 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Mon, 23 Feb 2015 20:39:46 -0800 Subject: [PATCH 1/3] add some fuse tests for readonly ipfs fs --- fuse/readonly/ipfs_test.go | 166 +++++++++++++++++++++++++++++++++++++ 1 file changed, 166 insertions(+) create mode 100644 fuse/readonly/ipfs_test.go diff --git a/fuse/readonly/ipfs_test.go b/fuse/readonly/ipfs_test.go new file mode 100644 index 000000000..8ede93c5e --- /dev/null +++ b/fuse/readonly/ipfs_test.go @@ -0,0 +1,166 @@ +package readonly + +import ( + "bytes" + "crypto/rand" + //context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" + "io/ioutil" + "os" + "testing" + + fstest "github.com/jbenet/go-ipfs/Godeps/_workspace/src/bazil.org/fuse/fs/fstestutil" + + core "github.com/jbenet/go-ipfs/core" + importer "github.com/jbenet/go-ipfs/importer" + chunk "github.com/jbenet/go-ipfs/importer/chunk" + dag "github.com/jbenet/go-ipfs/merkledag" + uio "github.com/jbenet/go-ipfs/unixfs/io" + u "github.com/jbenet/go-ipfs/util" + ci "github.com/jbenet/go-ipfs/util/testutil/ci" +) + +func maybeSkipFuseTests(t *testing.T) { + if ci.NoFuse() { + t.Skip("Skipping FUSE tests") + } +} + +func randBytes(size int) []byte { + b := make([]byte, size) + rand.Read(b) + return b +} + +func randObj(t *testing.T, nd *core.IpfsNode, size int64) (*dag.Node, []byte) { + buf := make([]byte, size) + u.NewTimeSeededRand().Read(buf) + read := bytes.NewReader(buf) + obj, err := importer.BuildTrickleDagFromReader(read, nd.DAG, nil, chunk.DefaultSplitter) + if err != nil { + t.Fatal(err) + } + + return obj, buf +} + +func setupIpfsTest(t *testing.T, node *core.IpfsNode) (*core.IpfsNode, *fstest.Mount) { + maybeSkipFuseTests(t) + + var err error + if node == nil { + node, err = core.NewMockNode() + if err != nil { + t.Fatal(err) + } + } + + fs := NewFileSystem(node) + mnt, err := fstest.MountedT(t, fs) + if err != nil { + t.Fatal(err) + } + + return node, mnt +} + +// Test writing an object and reading it back through fuse +func TestIpfsBasicRead(t *testing.T) { + if testing.Short() { + t.SkipNow() + } + nd, mnt := setupIpfsTest(t, nil) + defer mnt.Close() + + fi, data := randObj(t, nd, 10000) + k, err := fi.Key() + if err != nil { + t.Fatal(err) + } + + fname := mnt.Dir + "/" + k.String() + rbuf, err := ioutil.ReadFile(fname) + if err != nil { + t.Fatal(err) + } + + if !bytes.Equal(rbuf, data) { + t.Fatal("Incorrect Read!") + } +} + +// Test writing a file and reading it back +func TestIpfsBasicDirRead(t *testing.T) { + if testing.Short() { + t.SkipNow() + } + nd, mnt := setupIpfsTest(t, nil) + defer mnt.Close() + + // Make a 'file' + fi, data := randObj(t, nd, 10000) + k, err := fi.Key() + if err != nil { + t.Fatal(err) + } + + // Make a directory and put that file in it + db := uio.NewDirectory(nd.DAG) + err = db.AddChild("actual", k) + if err != nil { + t.Fatal(err) + } + + d1nd := db.GetNode() + d1ndk, err := nd.DAG.Add(d1nd) + if err != nil { + t.Fatal(err) + } + + dirname := mnt.Dir + "/" + d1ndk.String() + fname := dirname + "/actual" + rbuf, err := ioutil.ReadFile(fname) + if err != nil { + t.Fatal(err) + } + + dirents, err := ioutil.ReadDir(dirname) + if err != nil { + t.Fatal(err) + } + if len(dirents) != 1 { + t.Fatal("Bad directory entry count") + } + if dirents[0].Name() != "actual" { + t.Fatal("Bad directory entry") + } + + if !bytes.Equal(rbuf, data) { + t.Fatal("Incorrect Read!") + } +} + +// Test to make sure the filesystem reports file sizes correctly +func TestFileSizeReporting(t *testing.T) { + if testing.Short() { + t.SkipNow() + } + nd, mnt := setupIpfsTest(t, nil) + defer mnt.Close() + + fi, data := randObj(t, nd, 10000) + k, err := fi.Key() + if err != nil { + t.Fatal(err) + } + + fname := mnt.Dir + "/" + k.String() + + finfo, err := os.Stat(fname) + if err != nil { + t.Fatal(err) + } + + if finfo.Size() != int64(len(data)) { + t.Fatal("Read incorrect size from stat!") + } +} From 88b5a1a019a2913c6a60b5eeed560d5bfef1c6aa Mon Sep 17 00:00:00 2001 From: Jeromy Date: Mon, 23 Feb 2015 21:56:41 -0800 Subject: [PATCH 2/3] path.Join for cross platform goodness --- fuse/readonly/ipfs_test.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/fuse/readonly/ipfs_test.go b/fuse/readonly/ipfs_test.go index 8ede93c5e..5edd2f40e 100644 --- a/fuse/readonly/ipfs_test.go +++ b/fuse/readonly/ipfs_test.go @@ -6,6 +6,7 @@ import ( //context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" "io/ioutil" "os" + "path" "testing" fstest "github.com/jbenet/go-ipfs/Godeps/_workspace/src/bazil.org/fuse/fs/fstestutil" @@ -77,7 +78,7 @@ func TestIpfsBasicRead(t *testing.T) { t.Fatal(err) } - fname := mnt.Dir + "/" + k.String() + fname := path.Join(mnt.Dir, k.String()) rbuf, err := ioutil.ReadFile(fname) if err != nil { t.Fatal(err) @@ -116,8 +117,8 @@ func TestIpfsBasicDirRead(t *testing.T) { t.Fatal(err) } - dirname := mnt.Dir + "/" + d1ndk.String() - fname := dirname + "/actual" + dirname := path.Join(mnt.Dir, d1ndk.String()) + fname := path.Join(dirname, "actual") rbuf, err := ioutil.ReadFile(fname) if err != nil { t.Fatal(err) @@ -153,7 +154,7 @@ func TestFileSizeReporting(t *testing.T) { t.Fatal(err) } - fname := mnt.Dir + "/" + k.String() + fname := path.Join(mnt.Dir, k.String()) finfo, err := os.Stat(fname) if err != nil { From 36a21561f772584f8c2e7966c978d8bbf8347e98 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Mon, 23 Feb 2015 23:33:38 -0800 Subject: [PATCH 3/3] stress test for ipfs fuse --- core/mock.go | 2 + fuse/readonly/ipfs_test.go | 110 ++++++++++++++++++++++++++++++++++--- 2 files changed, 104 insertions(+), 8 deletions(-) diff --git a/core/mock.go b/core/mock.go index 2044f4363..987ba02b0 100644 --- a/core/mock.go +++ b/core/mock.go @@ -1,6 +1,7 @@ package core import ( + ctxgroup "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-ctxgroup" "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore" syncds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/sync" context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context" @@ -39,6 +40,7 @@ func NewMockNode() (*IpfsNode, error) { nd.Peerstore = peer.NewPeerstore() nd.Peerstore.AddPrivKey(p, ident.PrivateKey()) nd.Peerstore.AddPubKey(p, ident.PublicKey()) + nd.ContextGroup = ctxgroup.WithContext(ctx) nd.PeerHost, err = mocknet.New(ctx).AddPeer(ident.PrivateKey(), ident.Address()) // effectively offline if err != nil { diff --git a/fuse/readonly/ipfs_test.go b/fuse/readonly/ipfs_test.go index 5edd2f40e..ebdfac56f 100644 --- a/fuse/readonly/ipfs_test.go +++ b/fuse/readonly/ipfs_test.go @@ -2,16 +2,18 @@ package readonly import ( "bytes" - "crypto/rand" - //context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" + "fmt" "io/ioutil" + "math/rand" "os" "path" + "sync" "testing" fstest "github.com/jbenet/go-ipfs/Godeps/_workspace/src/bazil.org/fuse/fs/fstestutil" core "github.com/jbenet/go-ipfs/core" + coreunix "github.com/jbenet/go-ipfs/core/coreunix" importer "github.com/jbenet/go-ipfs/importer" chunk "github.com/jbenet/go-ipfs/importer/chunk" dag "github.com/jbenet/go-ipfs/merkledag" @@ -26,12 +28,6 @@ func maybeSkipFuseTests(t *testing.T) { } } -func randBytes(size int) []byte { - b := make([]byte, size) - rand.Read(b) - return b -} - func randObj(t *testing.T, nd *core.IpfsNode, size int64) (*dag.Node, []byte) { buf := make([]byte, size) u.NewTimeSeededRand().Read(buf) @@ -89,6 +85,104 @@ func TestIpfsBasicRead(t *testing.T) { } } +func getPaths(t *testing.T, ipfs *core.IpfsNode, name string, n *dag.Node) []string { + if len(n.Links) == 0 { + return []string{name} + } + var out []string + for _, lnk := range n.Links { + child, err := lnk.GetNode(ipfs.DAG) + if err != nil { + t.Fatal(err) + } + sub := getPaths(t, ipfs, path.Join(name, lnk.Name), child) + out = append(out, sub...) + } + return out +} + +// Perform a large number of concurrent reads to stress the system +func TestIpfsStressRead(t *testing.T) { + if testing.Short() { + t.SkipNow() + } + nd, mnt := setupIpfsTest(t, nil) + defer mnt.Close() + + var ks []u.Key + var paths []string + + nobj := 50 + ndiriter := 50 + + // Make a bunch of objects + for i := 0; i < nobj; i++ { + fi, _ := randObj(t, nd, rand.Int63n(50000)) + k, err := fi.Key() + if err != nil { + t.Fatal(err) + } + + ks = append(ks, k) + paths = append(paths, k.String()) + } + + // Now make a bunch of dirs + for i := 0; i < ndiriter; i++ { + db := uio.NewDirectory(nd.DAG) + for j := 0; j < 1+rand.Intn(10); j++ { + name := fmt.Sprintf("child%d", j) + err := db.AddChild(name, ks[rand.Intn(len(ks))]) + if err != nil { + t.Fatal(err) + } + } + newdir := db.GetNode() + k, err := nd.DAG.Add(newdir) + if err != nil { + t.Fatal(err) + } + + ks = append(ks, k) + npaths := getPaths(t, nd, k.String(), newdir) + paths = append(paths, npaths...) + } + + // Now read a bunch, concurrently + wg := sync.WaitGroup{} + + for s := 0; s < 4; s++ { + wg.Add(1) + go func() { + defer wg.Done() + + for i := 0; i < 2000; i++ { + item := paths[rand.Intn(len(paths))] + fname := path.Join(mnt.Dir, item) + rbuf, err := ioutil.ReadFile(fname) + if err != nil { + t.Fatal(err) + } + + read, err := coreunix.Cat(nd, item) + if err != nil { + t.Fatal(err) + } + + data, err := ioutil.ReadAll(read) + if err != nil { + t.Fatal(err) + } + + if !bytes.Equal(rbuf, data) { + t.Fatal("Incorrect Read!") + } + } + }() + } + wg.Wait() +} + // Test writing a file and reading it back func TestIpfsBasicDirRead(t *testing.T) { if testing.Short() {