opensource/kubo
1
0
Fork 0
mirror of https://github.com/ipfs/kubo.git synced 2025-07-01 10:49:24 +08:00
Code Issues Projects Releases Wiki Activity

Don't waste 256KiB buffers on small chunks.

Browse Source 
License: MIT
Signed-off-by: Steven Allen <steven@stebalien.com>
This commit is contained in:
Steven Allen
2017-12-19 14:23:31 -08:00
parent ce22b83f24
commit aafbe65a13
2 changed files with 22 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
importer/chunk/splitting.go
View File

@ -5,6 +5,7 @@ import (
"io" "io"
logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log" logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log"
mpool "gx/ipfs/QmWBug6eBS7AxRdCDVuSY5CnSit7cS2XnPFYJWqWDumhCG/go-msgio/mpool"
) )
var log = logging.Logger("chunk") var log = logging.Logger("chunk")
@ -51,14 +52,14 @@ func Chan(s Splitter) (<-chan []byte, <-chan error) {
type sizeSplitterv2 struct { type sizeSplitterv2 struct {
r io.Reader r io.Reader
size int64 size uint32
err error err error
} }
func NewSizeSplitter(r io.Reader, size int64) Splitter { func NewSizeSplitter(r io.Reader, size int64) Splitter {
return &sizeSplitterv2{ return &sizeSplitterv2{
r: r, r: r,
size: size, size: uint32(size),
} }
} }
@ -66,17 +67,22 @@ func (ss *sizeSplitterv2) NextBytes() ([]byte, error) {
if ss.err != nil { if ss.err != nil {
return nil, ss.err return nil, ss.err
} }
buf := make([]byte, ss.size)
n, err := io.ReadFull(ss.r, buf) full := mpool.ByteSlicePool.Get(ss.size).([]byte)[:ss.size]
if err == io.ErrUnexpectedEOF { n, err := io.ReadFull(ss.r, full)
switch err {
case io.ErrUnexpectedEOF:
ss.err = io.EOF ss.err = io.EOF
err = nil small := make([]byte, n)
} copy(small, full)
if err != nil { mpool.ByteSlicePool.Put(ss.size, full)
return small, nil
case nil:
return full, nil
default:
mpool.ByteSlicePool.Put(ss.size, full)
return nil, err return nil, err
} }
return buf[:n], nil
} }
func (ss *sizeSplitterv2) Reader() io.Reader { func (ss *sizeSplitterv2) Reader() io.Reader {

6
package.json
View File

@ -509,6 +509,12 @@
"hash": "QmYmhgAcvmDGXct1qBvc1kz9BxQSit1XBrTeiGZp2FvRyn", "hash": "QmYmhgAcvmDGXct1qBvc1kz9BxQSit1XBrTeiGZp2FvRyn",
"name": "go-libp2p-blankhost", "name": "go-libp2p-blankhost",
"version": "0.2.3" "version": "0.2.3"
},
{
"author": "jbenet",
"hash": "QmWBug6eBS7AxRdCDVuSY5CnSit7cS2XnPFYJWqWDumhCG",
"name": "go-msgio",
"version": "0.0.3"
} }
], ],
"gxVersion": "0.10.0", "gxVersion": "0.10.0",