opensource/kubo
1
0
Fork 0
mirror of https://github.com/ipfs/kubo.git synced 2025-06-29 01:12:24 +08:00
Code Issues Projects Releases Wiki Activity

Golint: unixfs/hamt

Browse Source 
Note, stuttering required renaming of HamtShard to Shard.

License: MIT
Signed-off-by: Hector Sanjuan <hector@protocol.ai>
This commit is contained in:
Hector Sanjuan
2018-02-06 12:43:30 +01:00
parent 694ab67952
commit 16934b9c68
3 changed files with 55 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

70
unixfs/hamt/hamt.go
View File

@ -38,10 +38,12 @@ import (
) )
const ( const (
// HashMurmur3 is the multiformats identifier for Murmur3
HashMurmur3 uint64 = 0x22 HashMurmur3 uint64 = 0x22
) )
type HamtShard struct { // A Shard represents the HAMT. It should be initialized with NewShard().
type Shard struct {
nd *dag.ProtoNode nd *dag.ProtoNode
bitfield *big.Int bitfield *big.Int
@ -66,9 +68,9 @@ type child interface {
Label() string Label() string
} }
// NewHamtShard creates a new, empty HAMT shard with the given size. // NewShard creates a new, empty HAMT shard with the given size.
func NewHamtShard(dserv ipld.DAGService, size int) (*HamtShard, error) { func NewShard(dserv ipld.DAGService, size int) (*Shard, error) {
ds, err := makeHamtShard(dserv, size) ds, err := makeShard(dserv, size)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -79,13 +81,13 @@ func NewHamtShard(dserv ipld.DAGService, size int) (*HamtShard, error) {
return ds, nil return ds, nil
} }
func makeHamtShard(ds ipld.DAGService, size int) (*HamtShard, error) { func makeShard(ds ipld.DAGService, size int) (*Shard, error) {
lg2s := int(math.Log2(float64(size))) lg2s := int(math.Log2(float64(size)))
if 1<<uint(lg2s) != size { if 1<<uint(lg2s) != size {
return nil, fmt.Errorf("hamt size should be a power of two") return nil, fmt.Errorf("hamt size should be a power of two")
} }
maxpadding := fmt.Sprintf("%X", size-1) maxpadding := fmt.Sprintf("%X", size-1)
return &HamtShard{ return &Shard{
tableSizeLg2: lg2s, tableSizeLg2: lg2s,
prefixPadStr: fmt.Sprintf("%%0%dX", len(maxpadding)), prefixPadStr: fmt.Sprintf("%%0%dX", len(maxpadding)),
maxpadlen: len(maxpadding), maxpadlen: len(maxpadding),
@ -95,7 +97,7 @@ func makeHamtShard(ds ipld.DAGService, size int) (*HamtShard, error) {
} }
// NewHamtFromDag creates new a HAMT shard from the given DAG. // NewHamtFromDag creates new a HAMT shard from the given DAG.
func NewHamtFromDag(dserv ipld.DAGService, nd ipld.Node) (*HamtShard, error) { func NewHamtFromDag(dserv ipld.DAGService, nd ipld.Node) (*Shard, error) {
pbnd, ok := nd.(*dag.ProtoNode) pbnd, ok := nd.(*dag.ProtoNode)
if !ok { if !ok {
return nil, dag.ErrLinkNotFound return nil, dag.ErrLinkNotFound
@ -114,7 +116,7 @@ func NewHamtFromDag(dserv ipld.DAGService, nd ipld.Node) (*HamtShard, error) {
return nil, fmt.Errorf("only murmur3 supported as hash function") return nil, fmt.Errorf("only murmur3 supported as hash function")
} }
ds, err := makeHamtShard(dserv, int(pbd.GetFanout())) ds, err := makeShard(dserv, int(pbd.GetFanout()))
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -129,17 +131,17 @@ func NewHamtFromDag(dserv ipld.DAGService, nd ipld.Node) (*HamtShard, error) {
} }
// SetPrefix sets the CID Prefix // SetPrefix sets the CID Prefix
func (ds *HamtShard) SetPrefix(prefix *cid.Prefix) { func (ds *Shard) SetPrefix(prefix *cid.Prefix) {
ds.prefix = prefix ds.prefix = prefix
} }
// Prefix gets the CID Prefix, may be nil if unset // Prefix gets the CID Prefix, may be nil if unset
func (ds *HamtShard) Prefix() *cid.Prefix { func (ds *Shard) Prefix() *cid.Prefix {
return ds.prefix return ds.prefix
} }
// Node serializes the HAMT structure into a merkledag node with unixfs formatting // Node serializes the HAMT structure into a merkledag node with unixfs formatting
func (ds *HamtShard) Node() (ipld.Node, error) { func (ds *Shard) Node() (ipld.Node, error) {
out := new(dag.ProtoNode) out := new(dag.ProtoNode)
out.SetPrefix(ds.prefix) out.SetPrefix(ds.prefix)
@ -214,14 +216,14 @@ func hash(val []byte) []byte {
return h.Sum(nil) return h.Sum(nil)
} }
// Label for HamtShards is the empty string, this is used to differentiate them from // Label for Shards is the empty string, this is used to differentiate them from
// value entries // value entries
func (ds *HamtShard) Label() string { func (ds *Shard) Label() string {
return "" return ""
} }
// Set sets 'name' = nd in the HAMT // Set sets 'name' = nd in the HAMT
func (ds *HamtShard) Set(ctx context.Context, name string, nd ipld.Node) error { func (ds *Shard) Set(ctx context.Context, name string, nd ipld.Node) error {
hv := &hashBits{b: hash([]byte(name))} hv := &hashBits{b: hash([]byte(name))}
err := ds.dserv.Add(ctx, nd) err := ds.dserv.Add(ctx, nd)
if err != nil { if err != nil {
@ -238,13 +240,13 @@ func (ds *HamtShard) Set(ctx context.Context, name string, nd ipld.Node) error {
} }
// Remove deletes the named entry if it exists, this operation is idempotent. // Remove deletes the named entry if it exists, this operation is idempotent.
func (ds *HamtShard) Remove(ctx context.Context, name string) error { func (ds *Shard) Remove(ctx context.Context, name string) error {
hv := &hashBits{b: hash([]byte(name))} hv := &hashBits{b: hash([]byte(name))}
return ds.modifyValue(ctx, hv, name, nil) return ds.modifyValue(ctx, hv, name, nil)
} }
// Find searches for a child node by 'name' within this hamt // Find searches for a child node by 'name' within this hamt
func (ds *HamtShard) Find(ctx context.Context, name string) (*ipld.Link, error) { func (ds *Shard) Find(ctx context.Context, name string) (*ipld.Link, error) {
hv := &hashBits{b: hash([]byte(name))} hv := &hashBits{b: hash([]byte(name))}
var out *ipld.Link var out *ipld.Link
@ -262,7 +264,7 @@ func (ds *HamtShard) Find(ctx context.Context, name string) (*ipld.Link, error)
// getChild returns the i'th child of this shard. If it is cached in the // getChild returns the i'th child of this shard. If it is cached in the
// children array, it will return it from there. Otherwise, it loads the child // children array, it will return it from there. Otherwise, it loads the child
// node from disk. // node from disk.
func (ds *HamtShard) getChild(ctx context.Context, i int) (child, error) { func (ds *Shard) getChild(ctx context.Context, i int) (child, error) {
if i >= len(ds.children) || i < 0 { if i >= len(ds.children) || i < 0 {
return nil, fmt.Errorf("invalid index passed to getChild (likely corrupt bitfield)") return nil, fmt.Errorf("invalid index passed to getChild (likely corrupt bitfield)")
} }
@ -281,7 +283,7 @@ func (ds *HamtShard) getChild(ctx context.Context, i int) (child, error) {
// loadChild reads the i'th child node of this shard from disk and returns it // loadChild reads the i'th child node of this shard from disk and returns it
// as a 'child' interface // as a 'child' interface
func (ds *HamtShard) loadChild(ctx context.Context, i int) (child, error) { func (ds *Shard) loadChild(ctx context.Context, i int) (child, error) {
lnk := ds.nd.Links()[i] lnk := ds.nd.Links()[i]
if len(lnk.Name) < ds.maxpadlen { if len(lnk.Name) < ds.maxpadlen {
return nil, fmt.Errorf("invalid link name '%s'", lnk.Name) return nil, fmt.Errorf("invalid link name '%s'", lnk.Name)
@ -326,12 +328,12 @@ func (ds *HamtShard) loadChild(ctx context.Context, i int) (child, error) {
return c, nil return c, nil
} }
func (ds *HamtShard) setChild(i int, c child) { func (ds *Shard) setChild(i int, c child) {
ds.children[i] = c ds.children[i] = c
} }
// Link returns a merklelink to this shard node // Link returns a merklelink to this shard node
func (ds *HamtShard) Link() (*ipld.Link, error) { func (ds *Shard) Link() (*ipld.Link, error) {
nd, err := ds.Node() nd, err := ds.Node()
if err != nil { if err != nil {
return nil, err return nil, err
@ -345,7 +347,7 @@ func (ds *HamtShard) Link() (*ipld.Link, error) {
return ipld.MakeLink(nd) return ipld.MakeLink(nd)
} }
func (ds *HamtShard) insertChild(idx int, key string, lnk *ipld.Link) error { func (ds *Shard) insertChild(idx int, key string, lnk *ipld.Link) error {
if lnk == nil { if lnk == nil {
return os.ErrNotExist return os.ErrNotExist
} }
@ -364,7 +366,7 @@ func (ds *HamtShard) insertChild(idx int, key string, lnk *ipld.Link) error {
return nil return nil
} }
func (ds *HamtShard) rmChild(i int) error { func (ds *Shard) rmChild(i int) error {
if i < 0 || i >= len(ds.children) || i >= len(ds.nd.Links()) { if i < 0 || i >= len(ds.children) || i >= len(ds.nd.Links()) {
return fmt.Errorf("hamt: attempted to remove child with out of range index") return fmt.Errorf("hamt: attempted to remove child with out of range index")
} }
@ -378,7 +380,7 @@ func (ds *HamtShard) rmChild(i int) error {
return nil return nil
} }
func (ds *HamtShard) getValue(ctx context.Context, hv *hashBits, key string, cb func(*shardValue) error) error { func (ds *Shard) getValue(ctx context.Context, hv *hashBits, key string, cb func(*shardValue) error) error {
idx := hv.Next(ds.tableSizeLg2) idx := hv.Next(ds.tableSizeLg2)
if ds.bitfield.Bit(int(idx)) == 1 { if ds.bitfield.Bit(int(idx)) == 1 {
cindex := ds.indexForBitPos(idx) cindex := ds.indexForBitPos(idx)
@ -389,7 +391,7 @@ func (ds *HamtShard) getValue(ctx context.Context, hv *hashBits, key string, cb
} }
switch child := child.(type) { switch child := child.(type) {
case *HamtShard: case *Shard:
return child.getValue(ctx, hv, key, cb) return child.getValue(ctx, hv, key, cb)
case *shardValue: case *shardValue:
if child.key == key { if child.key == key {
@ -401,7 +403,8 @@ func (ds *HamtShard) getValue(ctx context.Context, hv *hashBits, key string, cb
return os.ErrNotExist return os.ErrNotExist
} }
func (ds *HamtShard) EnumLinks(ctx context.Context) ([]*ipld.Link, error) { // EnumLinks collects all links in the Shard.
func (ds *Shard) EnumLinks(ctx context.Context) ([]*ipld.Link, error) {
var links []*ipld.Link var links []*ipld.Link
err := ds.ForEachLink(ctx, func(l *ipld.Link) error { err := ds.ForEachLink(ctx, func(l *ipld.Link) error {
links = append(links, l) links = append(links, l)
@ -410,7 +413,8 @@ func (ds *HamtShard) EnumLinks(ctx context.Context) ([]*ipld.Link, error) {
return links, err return links, err
} }
func (ds *HamtShard) ForEachLink(ctx context.Context, f func(*ipld.Link) error) error { // ForEachLink walks the Shard and calls the given function.
func (ds *Shard) ForEachLink(ctx context.Context, f func(*ipld.Link) error) error {
return ds.walkTrie(ctx, func(sv *shardValue) error { return ds.walkTrie(ctx, func(sv *shardValue) error {
lnk := sv.val lnk := sv.val
lnk.Name = sv.key lnk.Name = sv.key
@ -419,7 +423,7 @@ func (ds *HamtShard) ForEachLink(ctx context.Context, f func(*ipld.Link) error)
}) })
} }
func (ds *HamtShard) walkTrie(ctx context.Context, cb func(*shardValue) error) error { func (ds *Shard) walkTrie(ctx context.Context, cb func(*shardValue) error) error {
for i := 0; i < ds.tableSize; i++ { for i := 0; i < ds.tableSize; i++ {
if ds.bitfield.Bit(i) == 0 { if ds.bitfield.Bit(i) == 0 {
continue continue
@ -440,7 +444,7 @@ func (ds *HamtShard) walkTrie(ctx context.Context, cb func(*shardValue) error) e
return err return err
} }
case *HamtShard: case *Shard:
err := c.walkTrie(ctx, cb) err := c.walkTrie(ctx, cb)
if err != nil { if err != nil {
return err return err
@ -452,7 +456,7 @@ func (ds *HamtShard) walkTrie(ctx context.Context, cb func(*shardValue) error) e
return nil return nil
} }
func (ds *HamtShard) modifyValue(ctx context.Context, hv *hashBits, key string, val *ipld.Link) error { func (ds *Shard) modifyValue(ctx context.Context, hv *hashBits, key string, val *ipld.Link) error {
idx := hv.Next(ds.tableSizeLg2) idx := hv.Next(ds.tableSizeLg2)
if ds.bitfield.Bit(idx) != 1 { if ds.bitfield.Bit(idx) != 1 {
@ -467,7 +471,7 @@ func (ds *HamtShard) modifyValue(ctx context.Context, hv *hashBits, key string,
} }
switch child := child.(type) { switch child := child.(type) {
case *HamtShard: case *Shard:
err := child.modifyValue(ctx, hv, key, val) err := child.modifyValue(ctx, hv, key, val)
if err != nil { if err != nil {
return err return err
@ -510,7 +514,7 @@ func (ds *HamtShard) modifyValue(ctx context.Context, hv *hashBits, key string,
} }
// replace value with another shard, one level deeper // replace value with another shard, one level deeper
ns, err := NewHamtShard(ds.dserv, ds.tableSize) ns, err := NewShard(ds.dserv, ds.tableSize)
if err != nil { if err != nil {
return err return err
} }
@ -540,7 +544,7 @@ func (ds *HamtShard) modifyValue(ctx context.Context, hv *hashBits, key string,
// indexForBitPos returns the index within the collapsed array corresponding to // indexForBitPos returns the index within the collapsed array corresponding to
// the given bit in the bitset. The collapsed array contains only one entry // the given bit in the bitset. The collapsed array contains only one entry
// per bit set in the bitfield, and this function is used to map the indices. // per bit set in the bitfield, and this function is used to map the indices.
func (ds *HamtShard) indexForBitPos(bp int) int { func (ds *Shard) indexForBitPos(bp int) int {
// TODO: an optimization could reuse the same 'mask' here and change the size // TODO: an optimization could reuse the same 'mask' here and change the size
// as needed. This isnt yet done as the bitset package doesnt make it easy // as needed. This isnt yet done as the bitset package doesnt make it easy
// to do. // to do.
@ -553,6 +557,6 @@ func (ds *HamtShard) indexForBitPos(bp int) int {
} }
// linkNamePrefix takes in the bitfield index of an entry and returns its hex prefix // linkNamePrefix takes in the bitfield index of an entry and returns its hex prefix
func (ds *HamtShard) linkNamePrefix(idx int) string { func (ds *Shard) linkNamePrefix(idx int) string {
return fmt.Sprintf(ds.prefixPadStr, idx) return fmt.Sprintf(ds.prefixPadStr, idx)
} }

12
unixfs/hamt/hamt_stress_test.go
View File

@ -94,7 +94,7 @@ func TestOrderConsistency(t *testing.T) {
} }
} }
func validateOpSetCompletion(t *testing.T, s *HamtShard, keep, temp []string) error { func validateOpSetCompletion(t *testing.T, s *Shard, keep, temp []string) error {
ctx := context.TODO() ctx := context.TODO()
for _, n := range keep { for _, n := range keep {
_, err := s.Find(ctx, n) _, err := s.Find(ctx, n)
@ -113,9 +113,9 @@ func validateOpSetCompletion(t *testing.T, s *HamtShard, keep, temp []string) er
return nil return nil
} }
func executeOpSet(t *testing.T, ds ipld.DAGService, width int, ops []testOp) (*HamtShard, error) { func executeOpSet(t *testing.T, ds ipld.DAGService, width int, ops []testOp) (*Shard, error) {
ctx := context.TODO() ctx := context.TODO()
s, err := NewHamtShard(ds, width) s, err := NewShard(ds, width)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -189,9 +189,9 @@ func genOpSet(seed int64, keep, temp []string) []testOp {
} }
// executes the given op set with a repl to allow easier debugging // executes the given op set with a repl to allow easier debugging
/*func debugExecuteOpSet(ds node.DAGService, width int, ops []testOp) (*HamtShard, error) { /*func debugExecuteOpSet(ds node.DAGService, width int, ops []testOp) (*Shard, error) {
s, err := NewHamtShard(ds, width) s, err := NewShard(ds, width)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -244,7 +244,7 @@ mainloop:
} }
case "restart": case "restart":
var err error var err error
s, err = NewHamtShard(ds, width) s, err = NewShard(ds, width)
if err != nil { if err != nil {
panic(err) panic(err)
} }

24
unixfs/hamt/hamt_test.go
View File

@ -26,14 +26,14 @@ func shuffle(seed int64, arr []string) {
} }
} }
func makeDir(ds ipld.DAGService, size int) ([]string, *HamtShard, error) { func makeDir(ds ipld.DAGService, size int) ([]string, *Shard, error) {
return makeDirWidth(ds, size, 256) return makeDirWidth(ds, size, 256)
} }
func makeDirWidth(ds ipld.DAGService, size, width int) ([]string, *HamtShard, error) { func makeDirWidth(ds ipld.DAGService, size, width int) ([]string, *Shard, error) {
ctx := context.Background() ctx := context.Background()
s, _ := NewHamtShard(ds, width) s, _ := NewShard(ds, width)
var dirs []string var dirs []string
for i := 0; i < size; i++ { for i := 0; i < size; i++ {
@ -54,7 +54,7 @@ func makeDirWidth(ds ipld.DAGService, size, width int) ([]string, *HamtShard, er
return dirs, s, nil return dirs, s, nil
} }
func assertLink(s *HamtShard, name string, found bool) error { func assertLink(s *Shard, name string, found bool) error {
_, err := s.Find(context.Background(), name) _, err := s.Find(context.Background(), name)
switch err { switch err {
case os.ErrNotExist: case os.ErrNotExist:
@ -74,7 +74,7 @@ func assertLink(s *HamtShard, name string, found bool) error {
} }
} }
func assertSerializationWorks(ds ipld.DAGService, s *HamtShard) error { func assertSerializationWorks(ds ipld.DAGService, s *Shard) error {
ctx, cancel := context.WithCancel(context.Background()) ctx, cancel := context.WithCancel(context.Background())
defer cancel() defer cancel()
nd, err := s.Node() nd, err := s.Node()
@ -141,7 +141,7 @@ func TestBasicSet(t *testing.T) {
func TestDirBuilding(t *testing.T) { func TestDirBuilding(t *testing.T) {
ds := mdtest.Mock() ds := mdtest.Mock()
_, _ = NewHamtShard(ds, 256) _, _ = NewShard(ds, 256)
_, s, err := makeDir(ds, 200) _, s, err := makeDir(ds, 200)
if err != nil { if err != nil {
@ -164,7 +164,7 @@ func TestDirBuilding(t *testing.T) {
func TestShardReload(t *testing.T) { func TestShardReload(t *testing.T) {
ds := mdtest.Mock() ds := mdtest.Mock()
_, _ = NewHamtShard(ds, 256) _, _ = NewShard(ds, 256)
ctx, cancel := context.WithCancel(context.Background()) ctx, cancel := context.WithCancel(context.Background())
defer cancel() defer cancel()
@ -307,7 +307,7 @@ func TestSetAfterMarshal(t *testing.T) {
func TestDuplicateAddShard(t *testing.T) { func TestDuplicateAddShard(t *testing.T) {
ds := mdtest.Mock() ds := mdtest.Mock()
dir, _ := NewHamtShard(ds, 256) dir, _ := NewShard(ds, 256)
nd := new(dag.ProtoNode) nd := new(dag.ProtoNode)
ctx := context.Background() ctx := context.Background()
@ -430,7 +430,7 @@ func TestRemoveElemsAfterMarshal(t *testing.T) {
func TestBitfieldIndexing(t *testing.T) { func TestBitfieldIndexing(t *testing.T) {
ds := mdtest.Mock() ds := mdtest.Mock()
s, _ := NewHamtShard(ds, 256) s, _ := NewShard(ds, 256)
set := func(i int) { set := func(i int) {
s.bitfield.SetBit(s.bitfield, i, 1) s.bitfield.SetBit(s.bitfield, i, 1)
@ -466,7 +466,7 @@ func TestSetHamtChild(t *testing.T) {
ctx := context.Background() ctx := context.Background()
ds := mdtest.Mock() ds := mdtest.Mock()
s, _ := NewHamtShard(ds, 256) s, _ := NewShard(ds, 256)
e := ft.EmptyDirNode() e := ft.EmptyDirNode()
ds.Add(ctx, e) ds.Add(ctx, e)
@ -527,7 +527,7 @@ func BenchmarkHAMTSet(b *testing.B) {
ctx := context.Background() ctx := context.Background()
ds := mdtest.Mock() ds := mdtest.Mock()
sh, _ := NewHamtShard(ds, 256) sh, _ := NewShard(ds, 256)
nd, err := sh.Node() nd, err := sh.Node()
if err != nil { if err != nil {
b.Fatal(err) b.Fatal(err)
@ -560,7 +560,7 @@ func BenchmarkHAMTSet(b *testing.B) {
} }
func TestHamtBadSize(t *testing.T) { func TestHamtBadSize(t *testing.T) {
_, err := NewHamtShard(nil, 7) _, err := NewShard(nil, 7)
if err == nil { if err == nil {
t.Fatal("should have failed to construct hamt with bad size") t.Fatal("should have failed to construct hamt with bad size")
} }