opensource/kubo
1
0
Fork 0
mirror of https://github.com/ipfs/kubo.git synced 2025-07-01 19:24:14 +08:00
Code Issues Projects Releases Wiki Activity

Merge pull request #4664 from ipfs/doc/golint-unixfs

Browse Source 
Golint-ify unixfs module
This commit is contained in:
Whyrusleeping
2018-02-07 12:47:57 -08:00
committed by GitHub
parent 12354cc652 f4f450d774
commit 95daef4ed1
15 changed files with 174 additions and 103 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
unixfs/archive/archive.go
View File

@ -1,3 +1,4 @@
// Package archive provides utilities to archive and compress a [Unixfs] DAG.
package archive
import (

4
unixfs/archive/tar/writer.go
View File

@ -1,3 +1,5 @@
// Package tar provides functionality to write a unixfs merkledag
// as a tar archive.
package tar
import (
@ -69,6 +71,7 @@ func (w *Writer) writeFile(nd *mdag.ProtoNode, pb *upb.Data, fpath string) error
return nil
}
// WriteNode adds a node to the archive.
func (w *Writer) WriteNode(nd ipld.Node, fpath string) error {
switch nd := nd.(type) {
case *mdag.ProtoNode:
@ -106,6 +109,7 @@ func (w *Writer) WriteNode(nd ipld.Node, fpath string) error {
}
}
// Close closes the tar writer.
func (w *Writer) Close() error {
return w.TarW.Close()
}

70
unixfs/hamt/hamt.go
View File

@ -38,10 +38,12 @@ import (
)
const (
// HashMurmur3 is the multiformats identifier for Murmur3
HashMurmur3 uint64 = 0x22
)
type HamtShard struct {
// A Shard represents the HAMT. It should be initialized with NewShard().
type Shard struct {
nd *dag.ProtoNode
bitfield *big.Int
@ -66,9 +68,9 @@ type child interface {
Label() string
}
// NewHamtShard creates a new, empty HAMT shard with the given size.
func NewHamtShard(dserv ipld.DAGService, size int) (*HamtShard, error) {
ds, err := makeHamtShard(dserv, size)
// NewShard creates a new, empty HAMT shard with the given size.
func NewShard(dserv ipld.DAGService, size int) (*Shard, error) {
ds, err := makeShard(dserv, size)
if err != nil {
return nil, err
}
@ -79,13 +81,13 @@ func NewHamtShard(dserv ipld.DAGService, size int) (*HamtShard, error) {
return ds, nil
}
func makeHamtShard(ds ipld.DAGService, size int) (*HamtShard, error) {
func makeShard(ds ipld.DAGService, size int) (*Shard, error) {
lg2s := int(math.Log2(float64(size)))
if 1<<uint(lg2s) != size {
return nil, fmt.Errorf("hamt size should be a power of two")
}
maxpadding := fmt.Sprintf("%X", size-1)
return &HamtShard{
return &Shard{
tableSizeLg2: lg2s,
prefixPadStr: fmt.Sprintf("%%0%dX", len(maxpadding)),
maxpadlen: len(maxpadding),
@ -95,7 +97,7 @@ func makeHamtShard(ds ipld.DAGService, size int) (*HamtShard, error) {
}
// NewHamtFromDag creates new a HAMT shard from the given DAG.
func NewHamtFromDag(dserv ipld.DAGService, nd ipld.Node) (*HamtShard, error) {
func NewHamtFromDag(dserv ipld.DAGService, nd ipld.Node) (*Shard, error) {
pbnd, ok := nd.(*dag.ProtoNode)
if !ok {
return nil, dag.ErrLinkNotFound
@ -114,7 +116,7 @@ func NewHamtFromDag(dserv ipld.DAGService, nd ipld.Node) (*HamtShard, error) {
return nil, fmt.Errorf("only murmur3 supported as hash function")
}
ds, err := makeHamtShard(dserv, int(pbd.GetFanout()))
ds, err := makeShard(dserv, int(pbd.GetFanout()))
if err != nil {
return nil, err
}
@ -129,17 +131,17 @@ func NewHamtFromDag(dserv ipld.DAGService, nd ipld.Node) (*HamtShard, error) {
}
// SetPrefix sets the CID Prefix
func (ds *HamtShard) SetPrefix(prefix *cid.Prefix) {
func (ds *Shard) SetPrefix(prefix *cid.Prefix) {
ds.prefix = prefix
}
// Prefix gets the CID Prefix, may be nil if unset
func (ds *HamtShard) Prefix() *cid.Prefix {
func (ds *Shard) Prefix() *cid.Prefix {
return ds.prefix
}
// Node serializes the HAMT structure into a merkledag node with unixfs formatting
func (ds *HamtShard) Node() (ipld.Node, error) {
func (ds *Shard) Node() (ipld.Node, error) {
out := new(dag.ProtoNode)
out.SetPrefix(ds.prefix)
@ -214,14 +216,14 @@ func hash(val []byte) []byte {
return h.Sum(nil)
}
// Label for HamtShards is the empty string, this is used to differentiate them from
// Label for Shards is the empty string, this is used to differentiate them from
// value entries
func (ds *HamtShard) Label() string {
func (ds *Shard) Label() string {
return ""
}
// Set sets 'name' = nd in the HAMT
func (ds *HamtShard) Set(ctx context.Context, name string, nd ipld.Node) error {
func (ds *Shard) Set(ctx context.Context, name string, nd ipld.Node) error {
hv := &hashBits{b: hash([]byte(name))}
err := ds.dserv.Add(ctx, nd)
if err != nil {
@ -238,13 +240,13 @@ func (ds *HamtShard) Set(ctx context.Context, name string, nd ipld.Node) error {
}
// Remove deletes the named entry if it exists, this operation is idempotent.
func (ds *HamtShard) Remove(ctx context.Context, name string) error {
func (ds *Shard) Remove(ctx context.Context, name string) error {
hv := &hashBits{b: hash([]byte(name))}
return ds.modifyValue(ctx, hv, name, nil)
}
// Find searches for a child node by 'name' within this hamt
func (ds *HamtShard) Find(ctx context.Context, name string) (*ipld.Link, error) {
func (ds *Shard) Find(ctx context.Context, name string) (*ipld.Link, error) {
hv := &hashBits{b: hash([]byte(name))}
var out *ipld.Link
@ -262,7 +264,7 @@ func (ds *HamtShard) Find(ctx context.Context, name string) (*ipld.Link, error)
// getChild returns the i'th child of this shard. If it is cached in the
// children array, it will return it from there. Otherwise, it loads the child
// node from disk.
func (ds *HamtShard) getChild(ctx context.Context, i int) (child, error) {
func (ds *Shard) getChild(ctx context.Context, i int) (child, error) {
if i >= len(ds.children) || i < 0 {
return nil, fmt.Errorf("invalid index passed to getChild (likely corrupt bitfield)")
}
@ -281,7 +283,7 @@ func (ds *HamtShard) getChild(ctx context.Context, i int) (child, error) {
// loadChild reads the i'th child node of this shard from disk and returns it
// as a 'child' interface
func (ds *HamtShard) loadChild(ctx context.Context, i int) (child, error) {
func (ds *Shard) loadChild(ctx context.Context, i int) (child, error) {
lnk := ds.nd.Links()[i]
if len(lnk.Name) < ds.maxpadlen {
return nil, fmt.Errorf("invalid link name '%s'", lnk.Name)
@ -326,12 +328,12 @@ func (ds *HamtShard) loadChild(ctx context.Context, i int) (child, error) {
return c, nil
}
func (ds *HamtShard) setChild(i int, c child) {
func (ds *Shard) setChild(i int, c child) {
ds.children[i] = c
}
// Link returns a merklelink to this shard node
func (ds *HamtShard) Link() (*ipld.Link, error) {
func (ds *Shard) Link() (*ipld.Link, error) {
nd, err := ds.Node()
if err != nil {
return nil, err
@ -345,7 +347,7 @@ func (ds *HamtShard) Link() (*ipld.Link, error) {
return ipld.MakeLink(nd)
}
func (ds *HamtShard) insertChild(idx int, key string, lnk *ipld.Link) error {
func (ds *Shard) insertChild(idx int, key string, lnk *ipld.Link) error {
if lnk == nil {
return os.ErrNotExist
}
@ -364,7 +366,7 @@ func (ds *HamtShard) insertChild(idx int, key string, lnk *ipld.Link) error {
return nil
}
func (ds *HamtShard) rmChild(i int) error {
func (ds *Shard) rmChild(i int) error {
if i < 0 || i >= len(ds.children) || i >= len(ds.nd.Links()) {
return fmt.Errorf("hamt: attempted to remove child with out of range index")
}
@ -378,7 +380,7 @@ func (ds *HamtShard) rmChild(i int) error {
return nil
}
func (ds *HamtShard) getValue(ctx context.Context, hv *hashBits, key string, cb func(*shardValue) error) error {
func (ds *Shard) getValue(ctx context.Context, hv *hashBits, key string, cb func(*shardValue) error) error {
idx := hv.Next(ds.tableSizeLg2)
if ds.bitfield.Bit(int(idx)) == 1 {
cindex := ds.indexForBitPos(idx)
@ -389,7 +391,7 @@ func (ds *HamtShard) getValue(ctx context.Context, hv *hashBits, key string, cb
}
switch child := child.(type) {
case *HamtShard:
case *Shard:
return child.getValue(ctx, hv, key, cb)
case *shardValue:
if child.key == key {
@ -401,7 +403,8 @@ func (ds *HamtShard) getValue(ctx context.Context, hv *hashBits, key string, cb
return os.ErrNotExist
}
func (ds *HamtShard) EnumLinks(ctx context.Context) ([]*ipld.Link, error) {
// EnumLinks collects all links in the Shard.
func (ds *Shard) EnumLinks(ctx context.Context) ([]*ipld.Link, error) {
var links []*ipld.Link
err := ds.ForEachLink(ctx, func(l *ipld.Link) error {
links = append(links, l)
@ -410,7 +413,8 @@ func (ds *HamtShard) EnumLinks(ctx context.Context) ([]*ipld.Link, error) {
return links, err
}
func (ds *HamtShard) ForEachLink(ctx context.Context, f func(*ipld.Link) error) error {
// ForEachLink walks the Shard and calls the given function.
func (ds *Shard) ForEachLink(ctx context.Context, f func(*ipld.Link) error) error {
return ds.walkTrie(ctx, func(sv *shardValue) error {
lnk := sv.val
lnk.Name = sv.key
@ -419,7 +423,7 @@ func (ds *HamtShard) ForEachLink(ctx context.Context, f func(*ipld.Link) error)
})
}
func (ds *HamtShard) walkTrie(ctx context.Context, cb func(*shardValue) error) error {
func (ds *Shard) walkTrie(ctx context.Context, cb func(*shardValue) error) error {
for i := 0; i < ds.tableSize; i++ {
if ds.bitfield.Bit(i) == 0 {
continue
@ -440,7 +444,7 @@ func (ds *HamtShard) walkTrie(ctx context.Context, cb func(*shardValue) error) e
return err
}
case *HamtShard:
case *Shard:
err := c.walkTrie(ctx, cb)
if err != nil {
return err
@ -452,7 +456,7 @@ func (ds *HamtShard) walkTrie(ctx context.Context, cb func(*shardValue) error) e
return nil
}
func (ds *HamtShard) modifyValue(ctx context.Context, hv *hashBits, key string, val *ipld.Link) error {
func (ds *Shard) modifyValue(ctx context.Context, hv *hashBits, key string, val *ipld.Link) error {
idx := hv.Next(ds.tableSizeLg2)
if ds.bitfield.Bit(idx) != 1 {
@ -467,7 +471,7 @@ func (ds *HamtShard) modifyValue(ctx context.Context, hv *hashBits, key string,
}
switch child := child.(type) {
case *HamtShard:
case *Shard:
err := child.modifyValue(ctx, hv, key, val)
if err != nil {
return err
@ -510,7 +514,7 @@ func (ds *HamtShard) modifyValue(ctx context.Context, hv *hashBits, key string,
}
// replace value with another shard, one level deeper
ns, err := NewHamtShard(ds.dserv, ds.tableSize)
ns, err := NewShard(ds.dserv, ds.tableSize)
if err != nil {
return err
}
@ -540,7 +544,7 @@ func (ds *HamtShard) modifyValue(ctx context.Context, hv *hashBits, key string,
// indexForBitPos returns the index within the collapsed array corresponding to
// the given bit in the bitset. The collapsed array contains only one entry
// per bit set in the bitfield, and this function is used to map the indices.
func (ds *HamtShard) indexForBitPos(bp int) int {
func (ds *Shard) indexForBitPos(bp int) int {
// TODO: an optimization could reuse the same 'mask' here and change the size
// as needed. This isnt yet done as the bitset package doesnt make it easy
// to do.
@ -553,6 +557,6 @@ func (ds *HamtShard) indexForBitPos(bp int) int {
}
// linkNamePrefix takes in the bitfield index of an entry and returns its hex prefix
func (ds *HamtShard) linkNamePrefix(idx int) string {
func (ds *Shard) linkNamePrefix(idx int) string {
return fmt.Sprintf(ds.prefixPadStr, idx)
}

12
unixfs/hamt/hamt_stress_test.go
View File

@ -94,7 +94,7 @@ func TestOrderConsistency(t *testing.T) {
}
}
func validateOpSetCompletion(t *testing.T, s *HamtShard, keep, temp []string) error {
func validateOpSetCompletion(t *testing.T, s *Shard, keep, temp []string) error {
ctx := context.TODO()
for _, n := range keep {
_, err := s.Find(ctx, n)
@ -113,9 +113,9 @@ func validateOpSetCompletion(t *testing.T, s *HamtShard, keep, temp []string) er
return nil
}
func executeOpSet(t *testing.T, ds ipld.DAGService, width int, ops []testOp) (*HamtShard, error) {
func executeOpSet(t *testing.T, ds ipld.DAGService, width int, ops []testOp) (*Shard, error) {
ctx := context.TODO()
s, err := NewHamtShard(ds, width)
s, err := NewShard(ds, width)
if err != nil {
return nil, err
}
@ -189,9 +189,9 @@ func genOpSet(seed int64, keep, temp []string) []testOp {
}
// executes the given op set with a repl to allow easier debugging
/*func debugExecuteOpSet(ds node.DAGService, width int, ops []testOp) (*HamtShard, error) {
/*func debugExecuteOpSet(ds node.DAGService, width int, ops []testOp) (*Shard, error) {
s, err := NewHamtShard(ds, width)
s, err := NewShard(ds, width)
if err != nil {
return nil, err
}
@ -244,7 +244,7 @@ mainloop:
}
case "restart":
var err error
s, err = NewHamtShard(ds, width)
s, err = NewShard(ds, width)
if err != nil {
panic(err)
}

24
unixfs/hamt/hamt_test.go
View File

@ -26,14 +26,14 @@ func shuffle(seed int64, arr []string) {
}
}
func makeDir(ds ipld.DAGService, size int) ([]string, *HamtShard, error) {
func makeDir(ds ipld.DAGService, size int) ([]string, *Shard, error) {
return makeDirWidth(ds, size, 256)
}
func makeDirWidth(ds ipld.DAGService, size, width int) ([]string, *HamtShard, error) {
func makeDirWidth(ds ipld.DAGService, size, width int) ([]string, *Shard, error) {
ctx := context.Background()
s, _ := NewHamtShard(ds, width)
s, _ := NewShard(ds, width)
var dirs []string
for i := 0; i < size; i++ {
@ -54,7 +54,7 @@ func makeDirWidth(ds ipld.DAGService, size, width int) ([]string, *HamtShard, er
return dirs, s, nil
}
func assertLink(s *HamtShard, name string, found bool) error {
func assertLink(s *Shard, name string, found bool) error {
_, err := s.Find(context.Background(), name)
switch err {
case os.ErrNotExist:
@ -74,7 +74,7 @@ func assertLink(s *HamtShard, name string, found bool) error {
}
}
func assertSerializationWorks(ds ipld.DAGService, s *HamtShard) error {
func assertSerializationWorks(ds ipld.DAGService, s *Shard) error {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
nd, err := s.Node()
@ -141,7 +141,7 @@ func TestBasicSet(t *testing.T) {
func TestDirBuilding(t *testing.T) {
ds := mdtest.Mock()
_, _ = NewHamtShard(ds, 256)
_, _ = NewShard(ds, 256)
_, s, err := makeDir(ds, 200)
if err != nil {
@ -164,7 +164,7 @@ func TestDirBuilding(t *testing.T) {
func TestShardReload(t *testing.T) {
ds := mdtest.Mock()
_, _ = NewHamtShard(ds, 256)
_, _ = NewShard(ds, 256)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
@ -307,7 +307,7 @@ func TestSetAfterMarshal(t *testing.T) {
func TestDuplicateAddShard(t *testing.T) {
ds := mdtest.Mock()
dir, _ := NewHamtShard(ds, 256)
dir, _ := NewShard(ds, 256)
nd := new(dag.ProtoNode)
ctx := context.Background()
@ -430,7 +430,7 @@ func TestRemoveElemsAfterMarshal(t *testing.T) {
func TestBitfieldIndexing(t *testing.T) {
ds := mdtest.Mock()
s, _ := NewHamtShard(ds, 256)
s, _ := NewShard(ds, 256)
set := func(i int) {
s.bitfield.SetBit(s.bitfield, i, 1)
@ -466,7 +466,7 @@ func TestSetHamtChild(t *testing.T) {
ctx := context.Background()
ds := mdtest.Mock()
s, _ := NewHamtShard(ds, 256)
s, _ := NewShard(ds, 256)
e := ft.EmptyDirNode()
ds.Add(ctx, e)
@ -527,7 +527,7 @@ func BenchmarkHAMTSet(b *testing.B) {
ctx := context.Background()
ds := mdtest.Mock()
sh, _ := NewHamtShard(ds, 256)
sh, _ := NewShard(ds, 256)
nd, err := sh.Node()
if err != nil {
b.Fatal(err)
@ -560,7 +560,7 @@ func BenchmarkHAMTSet(b *testing.B) {
}
func TestHamtBadSize(t *testing.T) {
_, err := NewHamtShard(nil, 7)
_, err := NewShard(nil, 7)
if err == nil {
t.Fatal("should have failed to construct hamt with bad size")
}

24
unixfs/io/bufdagreader.go
View File

@ -6,25 +6,32 @@ import (
"io"
)
type bufDagReader struct {
// BufDagReader implements a DagReader that reads from a byte slice
// using a bytes.Reader. It is used for RawNodes.
type BufDagReader struct {
*bytes.Reader
}
func NewBufDagReader(b []byte) *bufDagReader {
return &bufDagReader{bytes.NewReader(b)}
// NewBufDagReader returns a DAG reader for the given byte slice.
// BufDagReader is used to read RawNodes.
func NewBufDagReader(b []byte) *BufDagReader {
return &BufDagReader{bytes.NewReader(b)}
}
var _ DagReader = (*bufDagReader)(nil)
var _ DagReader = (*BufDagReader)(nil)
func (*bufDagReader) Close() error {
// Close is a nop.
func (*BufDagReader) Close() error {
return nil
}
func (rd *bufDagReader) CtxReadFull(ctx context.Context, b []byte) (int, error) {
// CtxReadFull reads the slice onto b.
func (rd *BufDagReader) CtxReadFull(ctx context.Context, b []byte) (int, error) {
return rd.Read(b)
}
func (rd *bufDagReader) Offset() int64 {
// Offset returns the current offset.
func (rd *BufDagReader) Offset() int64 {
of, err := rd.Seek(0, io.SeekCurrent)
if err != nil {
panic("this should never happen " + err.Error())
@ -32,7 +39,8 @@ func (rd *bufDagReader) Offset() int64 {
return of
}
func (rd *bufDagReader) Size() uint64 {
// Size returns the size of the buffer.
func (rd *BufDagReader) Size() uint64 {
s := rd.Reader.Size()
if s < 0 {
panic("size smaller than 0 (impossible!!)")

12
unixfs/io/dagreader.go
View File

@ -14,10 +14,15 @@ import (
ipld "gx/ipfs/Qme5bWv7wtjUNGsK2BNGVUFPKiuxWrsqrtvYwCLRw8YFES/go-ipld-format"
)
var ErrIsDir = errors.New("this dag node is a directory")
var ErrCantReadSymlinks = errors.New("cannot currently read symlinks")
// Common errors
var (
ErrIsDir = errors.New("this dag node is a directory")
ErrCantReadSymlinks = errors.New("cannot currently read symlinks")
)
// A DagReader provides read-only read and seek acess to a unixfs file.
// Different implementations of readers are used for the different
// types of unixfs/protobuf-encoded nodes.
type DagReader interface {
ReadSeekCloser
Size() uint64
@ -25,6 +30,7 @@ type DagReader interface {
Offset() int64
}
// A ReadSeekCloser implements interfaces to read, copy, seek and close.
type ReadSeekCloser interface {
io.Reader
io.Seeker

2
unixfs/io/dagreader_test.go
View File

@ -102,7 +102,7 @@ func TestSeekAndReadLarge(t *testing.T) {
t.Fatal("seeked read failed")
}
pbdr := reader.(*pbDagReader)
pbdr := reader.(*PBDagReader)
var count int
for i, p := range pbdr.promises {
if i > 20 && i < 30 {

16
unixfs/io/dirbuilder.go
View File

@ -8,8 +8,8 @@ import (
mdag "github.com/ipfs/go-ipfs/merkledag"
format "github.com/ipfs/go-ipfs/unixfs"
hamt "github.com/ipfs/go-ipfs/unixfs/hamt"
cid "gx/ipfs/QmcZfnkapfECQGcLZaf9B79NRg7cRa9EnZh4LSbkCzwNvY/go-cid"
cid "gx/ipfs/QmcZfnkapfECQGcLZaf9B79NRg7cRa9EnZh4LSbkCzwNvY/go-cid"
ipld "gx/ipfs/Qme5bWv7wtjUNGsK2BNGVUFPKiuxWrsqrtvYwCLRw8YFES/go-ipld-format"
)
@ -25,11 +25,14 @@ var UseHAMTSharding = false
// DefaultShardWidth is the default value used for hamt sharding width.
var DefaultShardWidth = 256
// Directory allows to work with UnixFS directory nodes, adding and removing
// children. It allows to work with different directory schemes,
// like the classic or the HAMT one.
type Directory struct {
dserv ipld.DAGService
dirnode *mdag.ProtoNode
shard *hamt.HamtShard
shard *hamt.Shard
}
// NewDirectory returns a Directory. It needs a DAGService to add the Children
@ -37,7 +40,7 @@ func NewDirectory(dserv ipld.DAGService) *Directory {
db := new(Directory)
db.dserv = dserv
if UseHAMTSharding {
s, err := hamt.NewHamtShard(dserv, DefaultShardWidth)
s, err := hamt.NewShard(dserv, DefaultShardWidth)
if err != nil {
panic(err) // will only panic if DefaultShardWidth is a bad value
}
@ -113,7 +116,7 @@ func (d *Directory) AddChild(ctx context.Context, name string, nd ipld.Node) err
}
func (d *Directory) switchToSharding(ctx context.Context) error {
s, err := hamt.NewHamtShard(d.dserv, DefaultShardWidth)
s, err := hamt.NewShard(d.dserv, DefaultShardWidth)
if err != nil {
return err
}
@ -136,6 +139,7 @@ func (d *Directory) switchToSharding(ctx context.Context) error {
return nil
}
// ForEachLink applies the given function to Links in the directory.
func (d *Directory) ForEachLink(ctx context.Context, f func(*ipld.Link) error) error {
if d.shard == nil {
for _, l := range d.dirnode.Links() {
@ -149,6 +153,7 @@ func (d *Directory) ForEachLink(ctx context.Context, f func(*ipld.Link) error) e
return d.shard.ForEachLink(ctx, f)
}
// Links returns the all the links in the directory node.
func (d *Directory) Links(ctx context.Context) ([]*ipld.Link, error) {
if d.shard == nil {
return d.dirnode.Links(), nil
@ -157,6 +162,8 @@ func (d *Directory) Links(ctx context.Context) ([]*ipld.Link, error) {
return d.shard.EnumLinks(ctx)
}
// Find returns the root node of the file named 'name' within this directory.
// In the case of HAMT-directories, it will traverse the tree.
func (d *Directory) Find(ctx context.Context, name string) (ipld.Node, error) {
if d.shard == nil {
lnk, err := d.dirnode.GetNodeLink(name)
@ -179,6 +186,7 @@ func (d *Directory) Find(ctx context.Context, name string) (ipld.Node, error) {
return lnk.GetNode(ctx, d.dserv)
}
// RemoveChild removes the child with the given name.
func (d *Directory) RemoveChild(ctx context.Context, name string) error {
if d.shard == nil {
return d.dirnode.RemoveNodeLink(name)

2
unixfs/io/doc.go
View File

@ -1,3 +1,3 @@
// package unixfs/io implements convenience objects for working with the ipfs
// Package io implements convenience objects for working with the ipfs
// unixfs data format.
package io

37
unixfs/io/pbdagreader.go
View File

@ -15,8 +15,8 @@ import (
ipld "gx/ipfs/Qme5bWv7wtjUNGsK2BNGVUFPKiuxWrsqrtvYwCLRw8YFES/go-ipld-format"
)
// DagReader provides a way to easily read the data contained in a dag.
type pbDagReader struct {
// PBDagReader provides a way to easily read the data contained in a dag.
type PBDagReader struct {
serv ipld.NodeGetter
// the node being read
@ -48,13 +48,13 @@ type pbDagReader struct {
cancel func()
}
var _ DagReader = (*pbDagReader)(nil)
var _ DagReader = (*PBDagReader)(nil)
// NewPBFileReader constructs a new PBFileReader.
func NewPBFileReader(ctx context.Context, n *mdag.ProtoNode, pb *ftpb.Data, serv ipld.NodeGetter) *pbDagReader {
func NewPBFileReader(ctx context.Context, n *mdag.ProtoNode, pb *ftpb.Data, serv ipld.NodeGetter) *PBDagReader {
fctx, cancel := context.WithCancel(ctx)
curLinks := getLinkCids(n)
return &pbDagReader{
return &PBDagReader{
node: n,
serv: serv,
buf: NewBufDagReader(pb.GetData()),
@ -68,7 +68,7 @@ func NewPBFileReader(ctx context.Context, n *mdag.ProtoNode, pb *ftpb.Data, serv
const preloadSize = 10
func (dr *pbDagReader) preloadNextNodes(ctx context.Context) {
func (dr *PBDagReader) preloadNextNodes(ctx context.Context) {
beg := dr.linkPosition
end := beg + preloadSize
if end >= len(dr.links) {
@ -82,7 +82,7 @@ func (dr *pbDagReader) preloadNextNodes(ctx context.Context) {
// precalcNextBuf follows the next link in line and loads it from the
// DAGService, setting the next buffer to read from
func (dr *pbDagReader) precalcNextBuf(ctx context.Context) error {
func (dr *PBDagReader) precalcNextBuf(ctx context.Context) error {
if dr.buf != nil {
dr.buf.Close() // Just to make sure
dr.buf = nil
@ -145,17 +145,17 @@ func getLinkCids(n ipld.Node) []*cid.Cid {
}
// Size return the total length of the data from the DAG structured file.
func (dr *pbDagReader) Size() uint64 {
func (dr *PBDagReader) Size() uint64 {
return dr.pbdata.GetFilesize()
}
// Read reads data from the DAG structured file
func (dr *pbDagReader) Read(b []byte) (int, error) {
func (dr *PBDagReader) Read(b []byte) (int, error) {
return dr.CtxReadFull(dr.ctx, b)
}
// CtxReadFull reads data from the DAG structured file
func (dr *pbDagReader) CtxReadFull(ctx context.Context, b []byte) (int, error) {
func (dr *PBDagReader) CtxReadFull(ctx context.Context, b []byte) (int, error) {
if dr.buf == nil {
if err := dr.precalcNextBuf(ctx); err != nil {
return 0, err
@ -189,7 +189,8 @@ func (dr *pbDagReader) CtxReadFull(ctx context.Context, b []byte) (int, error) {
}
}
func (dr *pbDagReader) WriteTo(w io.Writer) (int64, error) {
// WriteTo writes to the given writer.
func (dr *PBDagReader) WriteTo(w io.Writer) (int64, error) {
if dr.buf == nil {
if err := dr.precalcNextBuf(dr.ctx); err != nil {
return 0, err
@ -220,12 +221,14 @@ func (dr *pbDagReader) WriteTo(w io.Writer) (int64, error) {
}
}
func (dr *pbDagReader) Close() error {
// Close closes the reader.
func (dr *PBDagReader) Close() error {
dr.cancel()
return nil
}
func (dr *pbDagReader) Offset() int64 {
// Offset returns the current reader offset
func (dr *PBDagReader) Offset() int64 {
return dr.offset
}
@ -233,7 +236,7 @@ func (dr *pbDagReader) Offset() int64 {
// interface matches standard unix seek
// TODO: check if we can do relative seeks, to reduce the amount of dagreader
// recreations that need to happen.
func (dr *pbDagReader) Seek(offset int64, whence int) (int64, error) {
func (dr *PBDagReader) Seek(offset int64, whence int) (int64, error) {
switch whence {
case io.SeekStart:
if offset < 0 {
@ -259,11 +262,11 @@ func (dr *pbDagReader) Seek(offset int64, whence int) (int64, error) {
dr.linkPosition = 0
dr.offset = offset
return offset, nil
} else {
// skip past root block data
left -= int64(len(pb.Data))
}
// skip past root block data
left -= int64(len(pb.Data))
// iterate through links and find where we need to be
for i := 0; i < len(pb.Blocksizes); i++ {
if pb.Blocksizes[i] > uint64(left) {

19
unixfs/mod/dagmodifier.go
View File

@ -1,10 +1,11 @@
// Package mod provides DAG modification utilities to, for example,
// insert additional nodes in a unixfs DAG or truncate them.
package mod
import (
"bytes"
"context"
"errors"
"fmt"
"io"
chunk "github.com/ipfs/go-ipfs/importer/chunk"
@ -19,8 +20,12 @@ import (
ipld "gx/ipfs/Qme5bWv7wtjUNGsK2BNGVUFPKiuxWrsqrtvYwCLRw8YFES/go-ipld-format"
)
var ErrSeekFail = errors.New("failed to seek properly")
var ErrUnrecognizedWhence = errors.New("unrecognized whence")
// Common errors
var (
ErrSeekFail = errors.New("failed to seek properly")
ErrUnrecognizedWhence = errors.New("unrecognized whence")
ErrNotUnixfs = errors.New("dagmodifier only supports unixfs nodes (proto or raw)")
)
// 2MB
var writebufferSize = 1 << 21
@ -46,8 +51,6 @@ type DagModifier struct {
read uio.DagReader
}
var ErrNotUnixfs = fmt.Errorf("dagmodifier only supports unixfs nodes (proto or raw)")
// NewDagModifier returns a new DagModifier, the Cid prefix for newly
// created nodes will be inherted from the passed in node. If the Cid
// version if not 0 raw leaves will also be enabled. The Prefix and
@ -412,7 +415,7 @@ func (dm *DagModifier) readPrep() error {
return nil
}
// Read data from this dag starting at the current offset
// CtxReadFull reads data from this dag starting at the current offset
func (dm *DagModifier) CtxReadFull(ctx context.Context, b []byte) (int, error) {
err := dm.readPrep()
if err != nil {
@ -438,6 +441,8 @@ func (dm *DagModifier) HasChanges() bool {
return dm.wrBuf != nil
}
// Seek modifies the offset according to whence. See unixfs/io for valid whence
// values.
func (dm *DagModifier) Seek(offset int64, whence int) (int64, error) {
err := dm.Sync()
if err != nil {
@ -479,6 +484,8 @@ func (dm *DagModifier) Seek(offset int64, whence int) (int64, error) {
return int64(dm.curWrOff), nil
}
// Truncate truncates the current Node to 'size' and replaces it with the
// new one.
func (dm *DagModifier) Truncate(size int64) error {
err := dm.Sync()
if err != nil {

13
unixfs/test/utils.go
View File

@ -21,6 +21,7 @@ import (
ipld "gx/ipfs/Qme5bWv7wtjUNGsK2BNGVUFPKiuxWrsqrtvYwCLRw8YFES/go-ipld-format"
)
// SizeSplitterGen creates a generator.
func SizeSplitterGen(size int64) chunk.SplitterGen {
return func(r io.Reader) chunk.Splitter {
return chunk.NewSizeSplitter(r, size)
@ -41,10 +42,13 @@ type NodeOpts struct {
RawLeavesUsed bool
}
var UseProtoBufLeaves = NodeOpts{Prefix: mdag.V0CidPrefix()}
var UseRawLeaves = NodeOpts{Prefix: mdag.V0CidPrefix(), ForceRawLeaves: true, RawLeavesUsed: true}
var UseCidV1 = NodeOpts{Prefix: mdag.V1CidPrefix(), RawLeavesUsed: true}
var UseBlake2b256 NodeOpts
// Some shorthands for NodeOpts.
var (
UseProtoBufLeaves = NodeOpts{Prefix: mdag.V0CidPrefix()}
UseRawLeaves = NodeOpts{Prefix: mdag.V0CidPrefix(), ForceRawLeaves: true, RawLeavesUsed: true}
UseCidV1 = NodeOpts{Prefix: mdag.V1CidPrefix(), RawLeavesUsed: true}
UseBlake2b256 NodeOpts
)
func init() {
UseBlake2b256 = UseCidV1
@ -88,6 +92,7 @@ func GetRandomNode(t testing.TB, dserv ipld.DAGService, size int64, opts NodeOpt
return buf, node
}
// ArrComp checks if two byte slices are the same.
func ArrComp(a, b []byte) error {
if len(a) != len(b) {
return fmt.Errorf("Arrays differ in length. %d != %d", len(a), len(b))

41
unixfs/format.go → unixfs/unixfs.go
View File

@ -1,4 +1,4 @@
// Package format implements a data format for files in the IPFS filesystem It
// Package unixfs implements a data format for files in the IPFS filesystem It
// is not the only format in ipfs, but it is the one that the filesystem
// assumes
package unixfs
@ -6,11 +6,13 @@ package unixfs
import (
"errors"
proto "gx/ipfs/QmZ4Qi3GaRbjcx28Sme5eMH7RQjGkt8wHxt2a65oLaeFEV/gogo-protobuf/proto"
dag "github.com/ipfs/go-ipfs/merkledag"
pb "github.com/ipfs/go-ipfs/unixfs/pb"
proto "gx/ipfs/QmZ4Qi3GaRbjcx28Sme5eMH7RQjGkt8wHxt2a65oLaeFEV/gogo-protobuf/proto"
)
// Shorthands for protobuffer types
const (
TRaw = pb.Data_Raw
TFile = pb.Data_File
@ -20,10 +22,14 @@ const (
THAMTShard = pb.Data_HAMTShard
)
var ErrMalformedFileFormat = errors.New("malformed data in file format")
var ErrInvalidDirLocation = errors.New("found directory node in unexpected place")
var ErrUnrecognizedType = errors.New("unrecognized node type")
// Common errors
var (
ErrMalformedFileFormat = errors.New("malformed data in file format")
ErrInvalidDirLocation = errors.New("found directory node in unexpected place")
ErrUnrecognizedType = errors.New("unrecognized node type")
)
// FromBytes unmarshals a byte slice as protobuf Data.
func FromBytes(data []byte) (*pb.Data, error) {
pbdata := new(pb.Data)
err := proto.Unmarshal(data, pbdata)
@ -33,6 +39,8 @@ func FromBytes(data []byte) (*pb.Data, error) {
return pbdata, nil
}
// FilePBData creates a protobuf File with the given
// byte slice and returns the marshaled protobuf bytes representing it.
func FilePBData(data []byte, totalsize uint64) []byte {
pbfile := new(pb.Data)
typ := pb.Data_File
@ -98,6 +106,7 @@ func SymlinkData(path string) ([]byte, error) {
return out, nil
}
// UnwrapData unmarshals a protobuf messages and returns the contents.
func UnwrapData(data []byte) ([]byte, error) {
pbdata := new(pb.Data)
err := proto.Unmarshal(data, pbdata)
@ -107,6 +116,10 @@ func UnwrapData(data []byte) ([]byte, error) {
return pbdata.GetData(), nil
}
// DataSize returns the size of the contents in protobuf wrapped slice.
// For raw data it simply provides the length of it. For Data_Files, it
// will return the associated filesize. Note that Data_Directories will
// return an error.
func DataSize(data []byte) (uint64, error) {
pbdata := new(pb.Data)
err := proto.Unmarshal(data, pbdata)
@ -116,16 +129,17 @@ func DataSize(data []byte) (uint64, error) {
switch pbdata.GetType() {
case pb.Data_Directory:
return 0, errors.New("Cant get data size of directory!")
return 0, errors.New("Cant get data size of directory")
case pb.Data_File:
return pbdata.GetFilesize(), nil
case pb.Data_Raw:
return uint64(len(pbdata.GetData())), nil
default:
return 0, errors.New("Unrecognized node data type!")
return 0, errors.New("Unrecognized node data type")
}
}
// An FSNode represents a filesystem object.
type FSNode struct {
Data []byte
@ -139,6 +153,7 @@ type FSNode struct {
Type pb.Data_DataType
}
// FSNodeFromBytes unmarshal a protobuf message onto an FSNode.
func FSNodeFromBytes(b []byte) (*FSNode, error) {
pbn := new(pb.Data)
err := proto.Unmarshal(b, pbn)
@ -160,11 +175,13 @@ func (n *FSNode) AddBlockSize(s uint64) {
n.blocksizes = append(n.blocksizes, s)
}
// RemoveBlockSize removes the given child block's size.
func (n *FSNode) RemoveBlockSize(i int) {
n.subtotal -= n.blocksizes[i]
n.blocksizes = append(n.blocksizes[:i], n.blocksizes[i+1:]...)
}
// GetBytes marshals this node as a protobuf message.
func (n *FSNode) GetBytes() ([]byte, error) {
pbn := new(pb.Data)
pbn.Type = &n.Type
@ -180,16 +197,19 @@ func (n *FSNode) FileSize() uint64 {
return uint64(len(n.Data)) + n.subtotal
}
// NumChildren returns the number of child blocks of this node
func (n *FSNode) NumChildren() int {
return len(n.blocksizes)
}
// Metadata is used to store additional FSNode information.
type Metadata struct {
MimeType string
Size uint64
}
//MetadataFromBytes Unmarshals a protobuf message into Metadata.
// MetadataFromBytes Unmarshals a protobuf Data message into Metadata.
// The provided slice should have been encoded with BytesForMetadata().
func MetadataFromBytes(b []byte) (*Metadata, error) {
pbd := new(pb.Data)
err := proto.Unmarshal(b, pbd)
@ -210,12 +230,16 @@ func MetadataFromBytes(b []byte) (*Metadata, error) {
return md, nil
}
// Bytes marshals Metadata as a protobuf message of Metadata type.
func (m *Metadata) Bytes() ([]byte, error) {
pbm := new(pb.Metadata)
pbm.MimeType = &m.MimeType
return proto.Marshal(pbm)
}
// BytesForMetadata wraps the given Metadata as a profobuf message of Data type,
// setting the DataType to Metadata. The wrapped bytes are itself the
// result of calling m.Bytes().
func BytesForMetadata(m *Metadata) ([]byte, error) {
pbd := new(pb.Data)
pbd.Filesize = proto.Uint64(m.Size)
@ -230,6 +254,7 @@ func BytesForMetadata(m *Metadata) ([]byte, error) {
return proto.Marshal(pbd)
}
// EmptyDirNode creates an empty folder Protonode.
func EmptyDirNode() *dag.ProtoNode {
return dag.NodeWithData(FolderPBData())
}

0
unixfs/format_test.go → unixfs/unixfs_test.go
View File