package helpers import ( "fmt" "context" chunk "github.com/ipfs/go-ipfs/importer/chunk" dag "github.com/ipfs/go-ipfs/merkledag" ft "github.com/ipfs/go-ipfs/unixfs" ) // BlockSizeLimit specifies the maximum size an imported block can have. var BlockSizeLimit = 1048576 // 1 MB // rough estimates on expected sizes var roughDataBlockSize = chunk.DefaultBlockSize var roughLinkBlockSize = 1 << 13 // 8KB var roughLinkSize = 34 + 8 + 5 // sha256 multihash + size + no name + protobuf framing // DefaultLinksPerBlock governs how the importer decides how many links there // will be per block. This calculation is based on expected distributions of: // * the expected distribution of block sizes // * the expected distribution of link sizes // * desired access speed // For now, we use: // // var roughLinkBlockSize = 1 << 13 // 8KB // var roughLinkSize = 288 // sha256 + framing + name // var DefaultLinksPerBlock = (roughLinkBlockSize / roughLinkSize) // // See calc_test.go var DefaultLinksPerBlock = (roughLinkBlockSize / roughLinkSize) // ErrSizeLimitExceeded signals that a block is larger than BlockSizeLimit. var ErrSizeLimitExceeded = fmt.Errorf("object size limit exceeded") // UnixfsNode is a struct created to aid in the generation // of unixfs DAG trees type UnixfsNode struct { node *dag.ProtoNode ufmt *ft.FSNode } // NewUnixfsNode creates a new Unixfs node to represent a file func NewUnixfsNode() *UnixfsNode { return &UnixfsNode{ node: new(dag.ProtoNode), ufmt: &ft.FSNode{Type: ft.TFile}, } } // NewUnixfsBlock creates a new Unixfs node to represent a raw data block func NewUnixfsBlock() *UnixfsNode { return &UnixfsNode{ node: new(dag.ProtoNode), ufmt: &ft.FSNode{Type: ft.TRaw}, } } // NewUnixfsNodeFromDag reconstructs a Unixfs node from a given dag node func NewUnixfsNodeFromDag(nd *dag.ProtoNode) (*UnixfsNode, error) { mb, err := ft.FSNodeFromBytes(nd.Data()) if err != nil { return nil, err } return &UnixfsNode{ node: nd, ufmt: mb, }, nil } func (n *UnixfsNode) NumChildren() int { return n.ufmt.NumChildren() } func (n *UnixfsNode) GetChild(ctx context.Context, i int, ds dag.DAGService) (*UnixfsNode, error) { nd, err := n.node.Links()[i].GetNode(ctx, ds) if err != nil { return nil, err } pbn, ok := nd.(*dag.ProtoNode) if !ok { return nil, dag.ErrNotProtobuf } return NewUnixfsNodeFromDag(pbn) } // addChild will add the given UnixfsNode as a child of the receiver. // the passed in DagBuilderHelper is used to store the child node an // pin it locally so it doesnt get lost func (n *UnixfsNode) AddChild(child *UnixfsNode, db *DagBuilderHelper) error { n.ufmt.AddBlockSize(child.ufmt.FileSize()) childnode, err := child.GetDagNode() if err != nil { return err } // Add a link to this node without storing a reference to the memory // This way, we avoid nodes building up and consuming all of our RAM err = n.node.AddNodeLinkClean("", childnode) if err != nil { return err } _, err = db.batch.Add(childnode) if err != nil { return err } return nil } // Removes the child node at the given index func (n *UnixfsNode) RemoveChild(index int, dbh *DagBuilderHelper) { n.ufmt.RemoveBlockSize(index) n.node.SetLinks(append(n.node.Links()[:index], n.node.Links()[index+1:]...)) } func (n *UnixfsNode) SetData(data []byte) { n.ufmt.Data = data } // getDagNode fills out the proper formatting for the unixfs node // inside of a DAG node and returns the dag node func (n *UnixfsNode) GetDagNode() (*dag.ProtoNode, error) { data, err := n.ufmt.GetBytes() if err != nil { return nil, err } n.node.SetData(data) return n.node, nil }