opensource/kubo
1
0
Fork 0
mirror of https://github.com/ipfs/kubo.git synced 2025-06-29 17:36:38 +08:00
Code Issues Projects Releases Wiki Activity

merkledag: limit number of objects in a batch to prevent out of fd issues

Browse Source 
License: MIT
Signed-off-by: Jeromy <jeromyj@gmail.com>
This commit is contained in:
Jeromy
2017-03-06 17:01:35 -08:00
parent 60b94a2c4f
commit 31af335dc0
1 changed files with 14 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
merkledag/merkledag.go
View File

@ -68,7 +68,15 @@ func (n *dagService) Add(nd node.Node) (*cid.Cid, error) {
} }
func (n *dagService) Batch() *Batch { func (n *dagService) Batch() *Batch {
return &Batch{ds: n, MaxSize: 8 * 1024 * 1024} return &Batch{
ds: n,
MaxSize: 8 << 20,
// By default, only batch up to 128 nodes at a time.
// The current implementation of flatfs opens this many file
// descriptors at the same time for the optimized batch write.
MaxBlocks: 128,
}
} }
// Get retrieves a node from the dagService, fetching the block in the BlockService // Get retrieves a node from the dagService, fetching the block in the BlockService
@ -376,15 +384,16 @@ func (np *nodePromise) Get(ctx context.Context) (node.Node, error) {
type Batch struct { type Batch struct {
ds *dagService ds *dagService
blocks []blocks.Block blocks []blocks.Block
size int size int
MaxSize int MaxSize int
MaxBlocks int
} }
func (t *Batch) Add(nd node.Node) (*cid.Cid, error) { func (t *Batch) Add(nd node.Node) (*cid.Cid, error) {
t.blocks = append(t.blocks, nd) t.blocks = append(t.blocks, nd)
t.size += len(nd.RawData()) t.size += len(nd.RawData())
if t.size > t.MaxSize { if t.size > t.MaxSize || len(t.blocks) > t.MaxBlocks {
return nd.Cid(), t.Commit() return nd.Cid(), t.Commit()
} }
return nd.Cid(), nil return nd.Cid(), nil