mirror of
https://github.com/ipfs/kubo.git
synced 2025-08-01 08:15:43 +08:00
568 lines
9.6 KiB
Go
568 lines
9.6 KiB
Go
package hamt
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"math/rand"
|
|
"os"
|
|
"sort"
|
|
"testing"
|
|
"time"
|
|
|
|
dag "github.com/ipfs/go-ipfs/merkledag"
|
|
mdtest "github.com/ipfs/go-ipfs/merkledag/test"
|
|
dagutils "github.com/ipfs/go-ipfs/merkledag/utils"
|
|
ft "github.com/ipfs/go-ipfs/unixfs"
|
|
|
|
ipld "gx/ipfs/Qme5bWv7wtjUNGsK2BNGVUFPKiuxWrsqrtvYwCLRw8YFES/go-ipld-format"
|
|
)
|
|
|
|
func shuffle(seed int64, arr []string) {
|
|
r := rand.New(rand.NewSource(seed))
|
|
for i := 0; i < len(arr); i++ {
|
|
a := r.Intn(len(arr))
|
|
b := r.Intn(len(arr))
|
|
arr[a], arr[b] = arr[b], arr[a]
|
|
}
|
|
}
|
|
|
|
func makeDir(ds ipld.DAGService, size int) ([]string, *HamtShard, error) {
|
|
return makeDirWidth(ds, size, 256)
|
|
}
|
|
|
|
func makeDirWidth(ds ipld.DAGService, size, width int) ([]string, *HamtShard, error) {
|
|
ctx := context.Background()
|
|
|
|
s, _ := NewHamtShard(ds, width)
|
|
|
|
var dirs []string
|
|
for i := 0; i < size; i++ {
|
|
dirs = append(dirs, fmt.Sprintf("DIRNAME%d", i))
|
|
}
|
|
|
|
shuffle(time.Now().UnixNano(), dirs)
|
|
|
|
for i := 0; i < len(dirs); i++ {
|
|
nd := ft.EmptyDirNode()
|
|
ds.Add(ctx, nd)
|
|
err := s.Set(ctx, dirs[i], nd)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
}
|
|
|
|
return dirs, s, nil
|
|
}
|
|
|
|
func assertLink(s *HamtShard, name string, found bool) error {
|
|
_, err := s.Find(context.Background(), name)
|
|
switch err {
|
|
case os.ErrNotExist:
|
|
if found {
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
case nil:
|
|
if found {
|
|
return nil
|
|
}
|
|
|
|
return fmt.Errorf("expected not to find link named %s", name)
|
|
default:
|
|
return err
|
|
}
|
|
}
|
|
|
|
func assertSerializationWorks(ds ipld.DAGService, s *HamtShard) error {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
nd, err := s.Node()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
nds, err := NewHamtFromDag(ds, nd)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
linksA, err := s.EnumLinks(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
linksB, err := nds.EnumLinks(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if len(linksA) != len(linksB) {
|
|
return fmt.Errorf("links arrays are different sizes")
|
|
}
|
|
|
|
for i, a := range linksA {
|
|
b := linksB[i]
|
|
if a.Name != b.Name {
|
|
return fmt.Errorf("links names mismatch")
|
|
}
|
|
|
|
if a.Cid.String() != b.Cid.String() {
|
|
return fmt.Errorf("link hashes dont match")
|
|
}
|
|
|
|
if a.Size != b.Size {
|
|
return fmt.Errorf("link sizes not the same")
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func TestBasicSet(t *testing.T) {
|
|
ds := mdtest.Mock()
|
|
for _, w := range []int{128, 256, 512, 1024, 2048, 4096} {
|
|
t.Run(fmt.Sprintf("BasicSet%d", w), func(t *testing.T) {
|
|
names, s, err := makeDirWidth(ds, 1000, w)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ctx := context.Background()
|
|
|
|
for _, d := range names {
|
|
_, err := s.Find(ctx, d)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestDirBuilding(t *testing.T) {
|
|
ds := mdtest.Mock()
|
|
_, _ = NewHamtShard(ds, 256)
|
|
|
|
_, s, err := makeDir(ds, 200)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
nd, err := s.Node()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
//printDag(ds, nd, 0)
|
|
|
|
k := nd.Cid()
|
|
|
|
if k.String() != "QmY89TkSEVHykWMHDmyejSWFj9CYNtvzw4UwnT9xbc4Zjc" {
|
|
t.Fatalf("output didnt match what we expected (got %s)", k.String())
|
|
}
|
|
}
|
|
|
|
func TestShardReload(t *testing.T) {
|
|
ds := mdtest.Mock()
|
|
_, _ = NewHamtShard(ds, 256)
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
|
|
_, s, err := makeDir(ds, 200)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
nd, err := s.Node()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
nds, err := NewHamtFromDag(ds, nd)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
lnks, err := nds.EnumLinks(ctx)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
if len(lnks) != 200 {
|
|
t.Fatal("not enough links back")
|
|
}
|
|
|
|
_, err = nds.Find(ctx, "DIRNAME50")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
// Now test roundtrip marshal with no operations
|
|
|
|
nds, err = NewHamtFromDag(ds, nd)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
ond, err := nds.Node()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
outk := ond.Cid()
|
|
ndk := nd.Cid()
|
|
|
|
if !outk.Equals(ndk) {
|
|
printDiff(ds, nd.(*dag.ProtoNode), ond.(*dag.ProtoNode))
|
|
t.Fatal("roundtrip serialization failed")
|
|
}
|
|
}
|
|
|
|
func TestRemoveElems(t *testing.T) {
|
|
ds := mdtest.Mock()
|
|
dirs, s, err := makeDir(ds, 500)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ctx := context.Background()
|
|
|
|
for i := 0; i < 100; i++ {
|
|
err := s.Remove(ctx, fmt.Sprintf("NOTEXIST%d", rand.Int()))
|
|
if err != os.ErrNotExist {
|
|
t.Fatal("shouldnt be able to remove things that don't exist")
|
|
}
|
|
}
|
|
|
|
for _, d := range dirs {
|
|
_, err := s.Find(ctx, d)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
shuffle(time.Now().UnixNano(), dirs)
|
|
|
|
for _, d := range dirs {
|
|
err := s.Remove(ctx, d)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
nd, err := s.Node()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
if len(nd.Links()) > 0 {
|
|
t.Fatal("shouldnt have any links here")
|
|
}
|
|
|
|
err = s.Remove(ctx, "doesnt exist")
|
|
if err != os.ErrNotExist {
|
|
t.Fatal("expected error does not exist")
|
|
}
|
|
}
|
|
|
|
func TestSetAfterMarshal(t *testing.T) {
|
|
ds := mdtest.Mock()
|
|
_, s, err := makeDir(ds, 300)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ctx := context.Background()
|
|
|
|
nd, err := s.Node()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
nds, err := NewHamtFromDag(ds, nd)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
empty := ft.EmptyDirNode()
|
|
for i := 0; i < 100; i++ {
|
|
err := nds.Set(ctx, fmt.Sprintf("moredirs%d", i), empty)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
links, err := nds.EnumLinks(ctx)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
if len(links) != 400 {
|
|
t.Fatal("expected 400 links")
|
|
}
|
|
|
|
err = assertSerializationWorks(ds, nds)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func TestDuplicateAddShard(t *testing.T) {
|
|
ds := mdtest.Mock()
|
|
dir, _ := NewHamtShard(ds, 256)
|
|
nd := new(dag.ProtoNode)
|
|
ctx := context.Background()
|
|
|
|
err := dir.Set(ctx, "test", nd)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
err = dir.Set(ctx, "test", nd)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
lnks, err := dir.EnumLinks(ctx)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
if len(lnks) != 1 {
|
|
t.Fatal("expected only one link")
|
|
}
|
|
}
|
|
|
|
func TestLoadFailsFromNonShard(t *testing.T) {
|
|
ds := mdtest.Mock()
|
|
nd := ft.EmptyDirNode()
|
|
|
|
_, err := NewHamtFromDag(ds, nd)
|
|
if err == nil {
|
|
t.Fatal("expected dir shard creation to fail when given normal directory")
|
|
}
|
|
|
|
nd = new(dag.ProtoNode)
|
|
|
|
_, err = NewHamtFromDag(ds, nd)
|
|
if err == nil {
|
|
t.Fatal("expected dir shard creation to fail when given normal directory")
|
|
}
|
|
}
|
|
|
|
func TestFindNonExisting(t *testing.T) {
|
|
ds := mdtest.Mock()
|
|
_, s, err := makeDir(ds, 100)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ctx := context.Background()
|
|
|
|
for i := 0; i < 200; i++ {
|
|
_, err := s.Find(ctx, fmt.Sprintf("notfound%d", i))
|
|
if err != os.ErrNotExist {
|
|
t.Fatal("expected ErrNotExist")
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestRemoveElemsAfterMarshal(t *testing.T) {
|
|
ds := mdtest.Mock()
|
|
dirs, s, err := makeDir(ds, 30)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ctx := context.Background()
|
|
|
|
sort.Strings(dirs)
|
|
|
|
err = s.Remove(ctx, dirs[0])
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
out, err := s.Find(ctx, dirs[0])
|
|
if err == nil {
|
|
t.Fatal("expected error, got: ", out)
|
|
}
|
|
|
|
nd, err := s.Node()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
nds, err := NewHamtFromDag(ds, nd)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
_, err = nds.Find(ctx, dirs[0])
|
|
if err == nil {
|
|
t.Fatal("expected not to find ", dirs[0])
|
|
}
|
|
|
|
for _, d := range dirs[1:] {
|
|
_, err := nds.Find(ctx, d)
|
|
if err != nil {
|
|
t.Fatal("could not find expected link after unmarshaling")
|
|
}
|
|
}
|
|
|
|
for _, d := range dirs[1:] {
|
|
err := nds.Remove(ctx, d)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
links, err := nds.EnumLinks(ctx)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
if len(links) != 0 {
|
|
t.Fatal("expected all links to be removed")
|
|
}
|
|
|
|
err = assertSerializationWorks(ds, nds)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func TestBitfieldIndexing(t *testing.T) {
|
|
ds := mdtest.Mock()
|
|
s, _ := NewHamtShard(ds, 256)
|
|
|
|
set := func(i int) {
|
|
s.bitfield.SetBit(s.bitfield, i, 1)
|
|
}
|
|
|
|
assert := func(i int, val int) {
|
|
if s.indexForBitPos(i) != val {
|
|
t.Fatalf("expected index %d to be %d", i, val)
|
|
}
|
|
}
|
|
|
|
assert(50, 0)
|
|
set(4)
|
|
set(5)
|
|
set(60)
|
|
|
|
assert(10, 2)
|
|
set(3)
|
|
assert(10, 3)
|
|
assert(1, 0)
|
|
|
|
assert(100, 4)
|
|
set(50)
|
|
assert(45, 3)
|
|
set(100)
|
|
assert(100, 5)
|
|
}
|
|
|
|
// test adding a sharded directory node as the child of another directory node.
|
|
// if improperly implemented, the parent hamt may assume the child is a part of
|
|
// itself.
|
|
func TestSetHamtChild(t *testing.T) {
|
|
ctx := context.Background()
|
|
|
|
ds := mdtest.Mock()
|
|
s, _ := NewHamtShard(ds, 256)
|
|
|
|
e := ft.EmptyDirNode()
|
|
ds.Add(ctx, e)
|
|
|
|
err := s.Set(ctx, "bar", e)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
snd, err := s.Node()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
_, ns, err := makeDir(ds, 50)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
err = ns.Set(ctx, "foo", snd)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
nsnd, err := ns.Node()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
hs, err := NewHamtFromDag(ds, nsnd)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
err = assertLink(hs, "bar", false)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
err = assertLink(hs, "foo", true)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func printDiff(ds ipld.DAGService, a, b *dag.ProtoNode) {
|
|
diff, err := dagutils.Diff(context.TODO(), ds, a, b)
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
|
|
for _, d := range diff {
|
|
fmt.Println(d)
|
|
}
|
|
}
|
|
|
|
func BenchmarkHAMTSet(b *testing.B) {
|
|
ctx := context.Background()
|
|
|
|
ds := mdtest.Mock()
|
|
sh, _ := NewHamtShard(ds, 256)
|
|
nd, err := sh.Node()
|
|
if err != nil {
|
|
b.Fatal(err)
|
|
}
|
|
|
|
err = ds.Add(ctx, nd)
|
|
if err != nil {
|
|
b.Fatal(err)
|
|
}
|
|
ds.Add(ctx, ft.EmptyDirNode())
|
|
|
|
for i := 0; i < b.N; i++ {
|
|
s, err := NewHamtFromDag(ds, nd)
|
|
if err != nil {
|
|
b.Fatal(err)
|
|
}
|
|
|
|
err = s.Set(context.TODO(), fmt.Sprint(i), ft.EmptyDirNode())
|
|
if err != nil {
|
|
b.Fatal(err)
|
|
}
|
|
|
|
out, err := s.Node()
|
|
if err != nil {
|
|
b.Fatal(err)
|
|
}
|
|
|
|
nd = out
|
|
}
|
|
}
|
|
|
|
func TestHamtBadSize(t *testing.T) {
|
|
_, err := NewHamtShard(nil, 7)
|
|
if err == nil {
|
|
t.Fatal("should have failed to construct hamt with bad size")
|
|
}
|
|
}
|