mirror of
https://github.com/ipfs/kubo.git
synced 2025-09-12 07:55:07 +08:00
100 lines
2.6 KiB
Go
100 lines
2.6 KiB
Go
package helpers
|
|
|
|
import (
|
|
"fmt"
|
|
|
|
chunk "github.com/jbenet/go-ipfs/importer/chunk"
|
|
dag "github.com/jbenet/go-ipfs/merkledag"
|
|
"github.com/jbenet/go-ipfs/pin"
|
|
ft "github.com/jbenet/go-ipfs/unixfs"
|
|
)
|
|
|
|
// BlockSizeLimit specifies the maximum size an imported block can have.
|
|
var BlockSizeLimit = 1048576 // 1 MB
|
|
|
|
// rough estimates on expected sizes
|
|
var roughDataBlockSize = chunk.DefaultBlockSize
|
|
var roughLinkBlockSize = 1 << 13 // 8KB
|
|
var roughLinkSize = 258 + 8 + 5 // sha256 multihash + size + no name + protobuf framing
|
|
|
|
// DefaultLinksPerBlock governs how the importer decides how many links there
|
|
// will be per block. This calculation is based on expected distributions of:
|
|
// * the expected distribution of block sizes
|
|
// * the expected distribution of link sizes
|
|
// * desired access speed
|
|
// For now, we use:
|
|
//
|
|
// var roughLinkBlockSize = 1 << 13 // 8KB
|
|
// var roughLinkSize = 288 // sha256 + framing + name
|
|
// var DefaultLinksPerBlock = (roughLinkBlockSize / roughLinkSize)
|
|
//
|
|
// See calc_test.go
|
|
var DefaultLinksPerBlock = (roughLinkBlockSize / roughLinkSize)
|
|
|
|
// ErrSizeLimitExceeded signals that a block is larger than BlockSizeLimit.
|
|
var ErrSizeLimitExceeded = fmt.Errorf("object size limit exceeded")
|
|
|
|
// UnixfsNode is a struct created to aid in the generation
|
|
// of unixfs DAG trees
|
|
type UnixfsNode struct {
|
|
node *dag.Node
|
|
ufmt *ft.MultiBlock
|
|
}
|
|
|
|
func NewUnixfsNode() *UnixfsNode {
|
|
return &UnixfsNode{
|
|
node: new(dag.Node),
|
|
ufmt: new(ft.MultiBlock),
|
|
}
|
|
}
|
|
|
|
func (n *UnixfsNode) NumChildren() int {
|
|
return n.ufmt.NumChildren()
|
|
}
|
|
|
|
// addChild will add the given UnixfsNode as a child of the receiver.
|
|
// the passed in DagBuilderHelper is used to store the child node an
|
|
// pin it locally so it doesnt get lost
|
|
func (n *UnixfsNode) AddChild(child *UnixfsNode, db *DagBuilderHelper) error {
|
|
n.ufmt.AddBlockSize(child.ufmt.FileSize())
|
|
|
|
childnode, err := child.GetDagNode()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// Add a link to this node without storing a reference to the memory
|
|
// This way, we avoid nodes building up and consuming all of our RAM
|
|
err = n.node.AddNodeLinkClean("", childnode)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
childkey, err := db.dserv.Add(childnode)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// Pin the child node indirectly
|
|
if db.mp != nil {
|
|
db.mp.PinWithMode(childkey, pin.Indirect)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (n *UnixfsNode) setData(data []byte) {
|
|
n.ufmt.Data = data
|
|
}
|
|
|
|
// getDagNode fills out the proper formatting for the unixfs node
|
|
// inside of a DAG node and returns the dag node
|
|
func (n *UnixfsNode) GetDagNode() (*dag.Node, error) {
|
|
data, err := n.ufmt.GetBytes()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
n.node.Data = data
|
|
return n.node, nil
|
|
}
|