WIP: Extract: importers/chunk module as go-ipfs-chunker

License: MIT
Signed-off-by: Hector Sanjuan <hector@protocol.ai>

core/corehttp/gateway_handler.go

@@ -16,12 +16,12 @@ import (
 	coreapi "github.com/ipfs/go-ipfs/core/coreapi"
 	coreiface "github.com/ipfs/go-ipfs/core/coreapi/interface"
 	"github.com/ipfs/go-ipfs/importer"
-	chunk "github.com/ipfs/go-ipfs/importer/chunk"
 	dag "github.com/ipfs/go-ipfs/merkledag"
 	dagutils "github.com/ipfs/go-ipfs/merkledag/utils"
 	path "github.com/ipfs/go-ipfs/path"
 	ft "github.com/ipfs/go-ipfs/unixfs"
 	uio "github.com/ipfs/go-ipfs/unixfs/io"
+	chunk "gx/ipfs/QmQXcAyrC4VBu9ZBqxoCthPot9PNhb4Uiw6iBDfQXudZJd/go-ipfs-chunker"
 
 	humanize "gx/ipfs/QmPSBJL4momYnE7DcUyk2DVhD6rH488ZmHBGLbxNdhU44K/go-humanize"
 	routing "gx/ipfs/QmTiWLZ6Fo5j4KcTVutZJ5KWRRJrbxzmxA4td8NfEdrPh7/go-libp2p-routing"

core/coreunix/add.go

@@ -15,7 +15,6 @@ import (
 	core "github.com/ipfs/go-ipfs/core"
 	"github.com/ipfs/go-ipfs/exchange/offline"
 	balanced "github.com/ipfs/go-ipfs/importer/balanced"
-	"github.com/ipfs/go-ipfs/importer/chunk"
 	ihelper "github.com/ipfs/go-ipfs/importer/helpers"
 	trickle "github.com/ipfs/go-ipfs/importer/trickle"
 	dag "github.com/ipfs/go-ipfs/merkledag"
@@ -26,6 +25,7 @@ import (
 
 	ds "gx/ipfs/QmPpegoMqhAEqjncrzArm7KVWAkCm78rqL2DPuNjhPrshg/go-datastore"
 	syncds "gx/ipfs/QmPpegoMqhAEqjncrzArm7KVWAkCm78rqL2DPuNjhPrshg/go-datastore/sync"
+	chunk "gx/ipfs/QmQXcAyrC4VBu9ZBqxoCthPot9PNhb4Uiw6iBDfQXudZJd/go-ipfs-chunker"
 	logging "gx/ipfs/QmRb5jh8z2E8hMGN2tkvs1yHynUanqnZ3UeKwgN1i9P1F8/go-log"
 	cid "gx/ipfs/QmcZfnkapfECQGcLZaf9B79NRg7cRa9EnZh4LSbkCzwNvY/go-cid"
 	files "gx/ipfs/QmceUdzxkimdYsgtX733uNgzf1DLHyBKN6ehGSp85ayppM/go-ipfs-cmdkit/files"

core/coreunix/metadata_test.go

@@ -11,10 +11,10 @@ import (
 	core "github.com/ipfs/go-ipfs/core"
 	offline "github.com/ipfs/go-ipfs/exchange/offline"
 	importer "github.com/ipfs/go-ipfs/importer"
-	chunk "github.com/ipfs/go-ipfs/importer/chunk"
 	merkledag "github.com/ipfs/go-ipfs/merkledag"
 	ft "github.com/ipfs/go-ipfs/unixfs"
 	uio "github.com/ipfs/go-ipfs/unixfs/io"
+	chunk "gx/ipfs/QmQXcAyrC4VBu9ZBqxoCthPot9PNhb4Uiw6iBDfQXudZJd/go-ipfs-chunker"
 
 	u "gx/ipfs/QmNiJuT8Ja3hMVpBHXv3Q6dwmperaQ6JjLtpMQgMCD7xvx/go-ipfs-util"
 	ds "gx/ipfs/QmPpegoMqhAEqjncrzArm7KVWAkCm78rqL2DPuNjhPrshg/go-datastore"

fuse/readonly/ipfs_test.go

@@ -17,9 +17,9 @@ import (
 	coreunix "github.com/ipfs/go-ipfs/core/coreunix"
 	coremock "github.com/ipfs/go-ipfs/core/mock"
 	importer "github.com/ipfs/go-ipfs/importer"
-	chunk "github.com/ipfs/go-ipfs/importer/chunk"
 	dag "github.com/ipfs/go-ipfs/merkledag"
 	uio "github.com/ipfs/go-ipfs/unixfs/io"
+	chunk "gx/ipfs/QmQXcAyrC4VBu9ZBqxoCthPot9PNhb4Uiw6iBDfQXudZJd/go-ipfs-chunker"
 	ci "gx/ipfs/QmVvkK7s5imCiq3JVbL3pGfnhcCnf3LrFJPF4GE2sAoGZf/go-testutil/ci"
 
 	u "gx/ipfs/QmNiJuT8Ja3hMVpBHXv3Q6dwmperaQ6JjLtpMQgMCD7xvx/go-ipfs-util"

importer/balanced/balanced_test.go

@@ -9,11 +9,11 @@ import (
 	mrand "math/rand"
 	"testing"
 
-	chunk "github.com/ipfs/go-ipfs/importer/chunk"
 	h "github.com/ipfs/go-ipfs/importer/helpers"
 	dag "github.com/ipfs/go-ipfs/merkledag"
 	mdtest "github.com/ipfs/go-ipfs/merkledag/test"
 	uio "github.com/ipfs/go-ipfs/unixfs/io"
+	chunk "gx/ipfs/QmQXcAyrC4VBu9ZBqxoCthPot9PNhb4Uiw6iBDfQXudZJd/go-ipfs-chunker"
 
 	u "gx/ipfs/QmNiJuT8Ja3hMVpBHXv3Q6dwmperaQ6JjLtpMQgMCD7xvx/go-ipfs-util"
 	ipld "gx/ipfs/Qme5bWv7wtjUNGsK2BNGVUFPKiuxWrsqrtvYwCLRw8YFES/go-ipld-format"

importer/chunk/parse.go

@@ -1,79 +0,0 @@
-package chunk
-
-import (
-	"errors"
-	"fmt"
-	"io"
-	"strconv"
-	"strings"
-)
-
-// FromString returns a Splitter depending on the given string:
-// it supports "default" (""), "size-{size}", "rabin", "rabin-{blocksize}" and
-// "rabin-{min}-{avg}-{max}".
-func FromString(r io.Reader, chunker string) (Splitter, error) {
-	switch {
-	case chunker == "" || chunker == "default":
-		return DefaultSplitter(r), nil
-
-	case strings.HasPrefix(chunker, "size-"):
-		sizeStr := strings.Split(chunker, "-")[1]
-		size, err := strconv.Atoi(sizeStr)
-		if err != nil {
-			return nil, err
-		}
-		return NewSizeSplitter(r, int64(size)), nil
-
-	case strings.HasPrefix(chunker, "rabin"):
-		return parseRabinString(r, chunker)
-
-	default:
-		return nil, fmt.Errorf("unrecognized chunker option: %s", chunker)
-	}
-}
-
-func parseRabinString(r io.Reader, chunker string) (Splitter, error) {
-	parts := strings.Split(chunker, "-")
-	switch len(parts) {
-	case 1:
-		return NewRabin(r, uint64(DefaultBlockSize)), nil
-	case 2:
-		size, err := strconv.Atoi(parts[1])
-		if err != nil {
-			return nil, err
-		}
-		return NewRabin(r, uint64(size)), nil
-	case 4:
-		sub := strings.Split(parts[1], ":")
-		if len(sub) > 1 && sub[0] != "min" {
-			return nil, errors.New("first label must be min")
-		}
-		min, err := strconv.Atoi(sub[len(sub)-1])
-		if err != nil {
-			return nil, err
-		}
-
-		sub = strings.Split(parts[2], ":")
-		if len(sub) > 1 && sub[0] != "avg" {
-			log.Error("sub == ", sub)
-			return nil, errors.New("second label must be avg")
-		}
-		avg, err := strconv.Atoi(sub[len(sub)-1])
-		if err != nil {
-			return nil, err
-		}
-
-		sub = strings.Split(parts[3], ":")
-		if len(sub) > 1 && sub[0] != "max" {
-			return nil, errors.New("final label must be max")
-		}
-		max, err := strconv.Atoi(sub[len(sub)-1])
-		if err != nil {
-			return nil, err
-		}
-
-		return NewRabinMinMax(r, uint64(min), uint64(avg), uint64(max)), nil
-	default:
-		return nil, errors.New("incorrect format (expected 'rabin' 'rabin-[avg]' or 'rabin-[min]-[avg]-[max]'")
-	}
-}

importer/chunk/rabin.go

@@ -1,54 +0,0 @@
-package chunk
-
-import (
-	"hash/fnv"
-	"io"
-
-	"github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/whyrusleeping/chunker"
-)
-
-// IpfsRabinPoly is the irreducible polynomial of degree 53 used by for Rabin.
-var IpfsRabinPoly = chunker.Pol(17437180132763653)
-
-// Rabin implements the Splitter interface and splits content with Rabin
-// fingerprints.
-type Rabin struct {
-	r      *chunker.Chunker
-	reader io.Reader
-}
-
-// NewRabin creates a new Rabin splitter with the given
-// average block size.
-func NewRabin(r io.Reader, avgBlkSize uint64) *Rabin {
-	min := avgBlkSize / 3
-	max := avgBlkSize + (avgBlkSize / 2)
-
-	return NewRabinMinMax(r, min, avgBlkSize, max)
-}
-
-// NewRabinMinMax returns a new Rabin splitter which uses
-// the given min, average and max block sizes.
-func NewRabinMinMax(r io.Reader, min, avg, max uint64) *Rabin {
-	h := fnv.New32a()
-	ch := chunker.New(r, IpfsRabinPoly, h, avg, min, max)
-
-	return &Rabin{
-		r:      ch,
-		reader: r,
-	}
-}
-
-// NextBytes reads the next bytes from the reader and returns a slice.
-func (r *Rabin) NextBytes() ([]byte, error) {
-	ch, err := r.r.Next()
-	if err != nil {
-		return nil, err
-	}
-
-	return ch.Data, nil
-}
-
-// Reader returns the io.Reader associated to this Splitter.
-func (r *Rabin) Reader() io.Reader {
-	return r.reader
-}

importer/chunk/rabin_test.go

@@ -1,81 +0,0 @@
-package chunk
-
-import (
-	"bytes"
-	"fmt"
-	"io"
-	"testing"
-
-	util "gx/ipfs/QmNiJuT8Ja3hMVpBHXv3Q6dwmperaQ6JjLtpMQgMCD7xvx/go-ipfs-util"
-	blocks "gx/ipfs/Qmej7nf81hi2x2tvjRBF3mcp74sQyuDH4VMYDGd1YtXjb2/go-block-format"
-)
-
-func TestRabinChunking(t *testing.T) {
-	data := make([]byte, 1024*1024*16)
-	util.NewTimeSeededRand().Read(data)
-
-	r := NewRabin(bytes.NewReader(data), 1024*256)
-
-	var chunks [][]byte
-
-	for {
-		chunk, err := r.NextBytes()
-		if err != nil {
-			if err == io.EOF {
-				break
-			}
-			t.Fatal(err)
-		}
-
-		chunks = append(chunks, chunk)
-	}
-
-	fmt.Printf("average block size: %d\n", len(data)/len(chunks))
-
-	unchunked := bytes.Join(chunks, nil)
-	if !bytes.Equal(unchunked, data) {
-		fmt.Printf("%d %d\n", len(unchunked), len(data))
-		t.Fatal("data was chunked incorrectly")
-	}
-}
-
-func chunkData(t *testing.T, data []byte) map[string]blocks.Block {
-	r := NewRabin(bytes.NewReader(data), 1024*256)
-
-	blkmap := make(map[string]blocks.Block)
-
-	for {
-		blk, err := r.NextBytes()
-		if err != nil {
-			if err == io.EOF {
-				break
-			}
-			t.Fatal(err)
-		}
-
-		b := blocks.NewBlock(blk)
-		blkmap[b.Cid().KeyString()] = b
-	}
-
-	return blkmap
-}
-
-func TestRabinChunkReuse(t *testing.T) {
-	data := make([]byte, 1024*1024*16)
-	util.NewTimeSeededRand().Read(data)
-
-	ch1 := chunkData(t, data[1000:])
-	ch2 := chunkData(t, data)
-
-	var extra int
-	for k := range ch2 {
-		_, ok := ch1[k]
-		if !ok {
-			extra++
-		}
-	}
-
-	if extra > 2 {
-		t.Log("too many spare chunks made")
-	}
-}

importer/chunk/splitting.go

@@ -1,105 +0,0 @@
-// Package chunk implements streaming block splitters.
-// Splitters read data from a reader and provide byte slices (chunks)
-// The size and contents of these slices depend on the splitting method
-// used.
-package chunk
-
-import (
-	"io"
-
-	logging "gx/ipfs/QmRb5jh8z2E8hMGN2tkvs1yHynUanqnZ3UeKwgN1i9P1F8/go-log"
-	mpool "gx/ipfs/QmWBug6eBS7AxRdCDVuSY5CnSit7cS2XnPFYJWqWDumhCG/go-msgio/mpool"
-)
-
-var log = logging.Logger("chunk")
-
-// DefaultBlockSize is the chunk size that splitters produce (or aim to).
-var DefaultBlockSize int64 = 1024 * 256
-
-// A Splitter reads bytes from a Reader and creates "chunks" (byte slices)
-// that can be used to build DAG nodes.
-type Splitter interface {
-	Reader() io.Reader
-	NextBytes() ([]byte, error)
-}
-
-// SplitterGen is a splitter generator, given a reader.
-type SplitterGen func(r io.Reader) Splitter
-
-// DefaultSplitter returns a SizeSplitter with the DefaultBlockSize.
-func DefaultSplitter(r io.Reader) Splitter {
-	return NewSizeSplitter(r, DefaultBlockSize)
-}
-
-// SizeSplitterGen returns a SplitterGen function which will create
-// a splitter with the given size when called.
-func SizeSplitterGen(size int64) SplitterGen {
-	return func(r io.Reader) Splitter {
-		return NewSizeSplitter(r, size)
-	}
-}
-
-// Chan returns a channel that receives each of the chunks produced
-// by a splitter, along with another one for errors.
-func Chan(s Splitter) (<-chan []byte, <-chan error) {
-	out := make(chan []byte)
-	errs := make(chan error, 1)
-	go func() {
-		defer close(out)
-		defer close(errs)
-
-		// all-chunks loop (keep creating chunks)
-		for {
-			b, err := s.NextBytes()
-			if err != nil {
-				errs <- err
-				return
-			}
-
-			out <- b
-		}
-	}()
-	return out, errs
-}
-
-type sizeSplitterv2 struct {
-	r    io.Reader
-	size uint32
-	err  error
-}
-
-// NewSizeSplitter returns a new size-based Splitter with the given block size.
-func NewSizeSplitter(r io.Reader, size int64) Splitter {
-	return &sizeSplitterv2{
-		r:    r,
-		size: uint32(size),
-	}
-}
-
-// NextBytes produces a new chunk.
-func (ss *sizeSplitterv2) NextBytes() ([]byte, error) {
-	if ss.err != nil {
-		return nil, ss.err
-	}
-
-	full := mpool.ByteSlicePool.Get(ss.size).([]byte)[:ss.size]
-	n, err := io.ReadFull(ss.r, full)
-	switch err {
-	case io.ErrUnexpectedEOF:
-		ss.err = io.EOF
-		small := make([]byte, n)
-		copy(small, full)
-		mpool.ByteSlicePool.Put(ss.size, full)
-		return small, nil
-	case nil:
-		return full, nil
-	default:
-		mpool.ByteSlicePool.Put(ss.size, full)
-		return nil, err
-	}
-}
-
-// Reader returns the io.Reader associated to this Splitter.
-func (ss *sizeSplitterv2) Reader() io.Reader {
-	return ss.r
-}

importer/chunk/splitting_test.go

@@ -1,120 +0,0 @@
-package chunk
-
-import (
-	"bytes"
-	"io"
-	"testing"
-
-	u "gx/ipfs/QmNiJuT8Ja3hMVpBHXv3Q6dwmperaQ6JjLtpMQgMCD7xvx/go-ipfs-util"
-)
-
-func randBuf(t *testing.T, size int) []byte {
-	buf := make([]byte, size)
-	if _, err := u.NewTimeSeededRand().Read(buf); err != nil {
-		t.Fatal("failed to read enough randomness")
-	}
-	return buf
-}
-
-func copyBuf(buf []byte) []byte {
-	cpy := make([]byte, len(buf))
-	copy(cpy, buf)
-	return cpy
-}
-
-func TestSizeSplitterOverAllocate(t *testing.T) {
-	max := 1000
-	r := bytes.NewReader(randBuf(t, max))
-	chunksize := int64(1024 * 256)
-	splitter := NewSizeSplitter(r, chunksize)
-	chunk, err := splitter.NextBytes()
-	if err != nil {
-		t.Fatal(err)
-	}
-	if cap(chunk) > len(chunk) {
-		t.Fatal("chunk capacity too large")
-	}
-}
-
-func TestSizeSplitterIsDeterministic(t *testing.T) {
-	if testing.Short() {
-		t.SkipNow()
-	}
-
-	test := func() {
-		bufR := randBuf(t, 10000000) // crank this up to satisfy yourself.
-		bufA := copyBuf(bufR)
-		bufB := copyBuf(bufR)
-
-		chunksA, _ := Chan(DefaultSplitter(bytes.NewReader(bufA)))
-		chunksB, _ := Chan(DefaultSplitter(bytes.NewReader(bufB)))
-
-		for n := 0; ; n++ {
-			a, moreA := <-chunksA
-			b, moreB := <-chunksB
-
-			if !moreA {
-				if moreB {
-					t.Fatal("A ended, B didnt.")
-				}
-				return
-			}
-
-			if !bytes.Equal(a, b) {
-				t.Fatalf("chunk %d not equal", n)
-			}
-		}
-	}
-
-	for run := 0; run < 1; run++ { // crank this up to satisfy yourself.
-		test()
-	}
-}
-
-func TestSizeSplitterFillsChunks(t *testing.T) {
-	if testing.Short() {
-		t.SkipNow()
-	}
-
-	max := 10000000
-	b := randBuf(t, max)
-	r := &clipReader{r: bytes.NewReader(b), size: 4000}
-	chunksize := int64(1024 * 256)
-	c, _ := Chan(NewSizeSplitter(r, chunksize))
-
-	sofar := 0
-	whole := make([]byte, max)
-	for chunk := range c {
-
-		bc := b[sofar : sofar+len(chunk)]
-		if !bytes.Equal(bc, chunk) {
-			t.Fatalf("chunk not correct: (sofar: %d) %d != %d, %v != %v", sofar, len(bc), len(chunk), bc[:100], chunk[:100])
-		}
-
-		copy(whole[sofar:], chunk)
-
-		sofar += len(chunk)
-		if sofar != max && len(chunk) < int(chunksize) {
-			t.Fatal("sizesplitter split at a smaller size")
-		}
-	}
-
-	if !bytes.Equal(b, whole) {
-		t.Fatal("splitter did not split right")
-	}
-}
-
-type clipReader struct {
-	size int
-	r    io.Reader
-}
-
-func (s *clipReader) Read(buf []byte) (int, error) {
-
-	// clip the incoming buffer to produce smaller chunks
-	if len(buf) > s.size {
-		buf = buf[:s.size]
-	}
-
-	return s.r.Read(buf)
-}

importer/helpers/dagbuilder.go

@@ -5,10 +5,10 @@ import (
 	"io"
 	"os"
 
-	"github.com/ipfs/go-ipfs/importer/chunk"
 	dag "github.com/ipfs/go-ipfs/merkledag"
 	ft "github.com/ipfs/go-ipfs/unixfs"
 
+	chunk "gx/ipfs/QmQXcAyrC4VBu9ZBqxoCthPot9PNhb4Uiw6iBDfQXudZJd/go-ipfs-chunker"
 	cid "gx/ipfs/QmcZfnkapfECQGcLZaf9B79NRg7cRa9EnZh4LSbkCzwNvY/go-cid"
 	files "gx/ipfs/QmceUdzxkimdYsgtX733uNgzf1DLHyBKN6ehGSp85ayppM/go-ipfs-cmdkit/files"
 	ipld "gx/ipfs/Qme5bWv7wtjUNGsK2BNGVUFPKiuxWrsqrtvYwCLRw8YFES/go-ipld-format"

importer/importer.go

@@ -6,11 +6,11 @@ import (
 	"fmt"
 	"os"
 
+	chunk "gx/ipfs/QmQXcAyrC4VBu9ZBqxoCthPot9PNhb4Uiw6iBDfQXudZJd/go-ipfs-chunker"
 	"gx/ipfs/QmceUdzxkimdYsgtX733uNgzf1DLHyBKN6ehGSp85ayppM/go-ipfs-cmdkit/files"
 	ipld "gx/ipfs/Qme5bWv7wtjUNGsK2BNGVUFPKiuxWrsqrtvYwCLRw8YFES/go-ipld-format"
 
 	bal "github.com/ipfs/go-ipfs/importer/balanced"
-	"github.com/ipfs/go-ipfs/importer/chunk"
 	h "github.com/ipfs/go-ipfs/importer/helpers"
 	trickle "github.com/ipfs/go-ipfs/importer/trickle"
 )

importer/importer_test.go

@@ -7,9 +7,9 @@ import (
 	"io/ioutil"
 	"testing"
 
-	chunk "github.com/ipfs/go-ipfs/importer/chunk"
 	mdtest "github.com/ipfs/go-ipfs/merkledag/test"
 	uio "github.com/ipfs/go-ipfs/unixfs/io"
+	chunk "gx/ipfs/QmQXcAyrC4VBu9ZBqxoCthPot9PNhb4Uiw6iBDfQXudZJd/go-ipfs-chunker"
 
 	u "gx/ipfs/QmNiJuT8Ja3hMVpBHXv3Q6dwmperaQ6JjLtpMQgMCD7xvx/go-ipfs-util"
 	ipld "gx/ipfs/Qme5bWv7wtjUNGsK2BNGVUFPKiuxWrsqrtvYwCLRw8YFES/go-ipld-format"

importer/trickle/trickle_test.go

@@ -9,12 +9,12 @@ import (
 	mrand "math/rand"
 	"testing"
 
-	chunk "github.com/ipfs/go-ipfs/importer/chunk"
 	h "github.com/ipfs/go-ipfs/importer/helpers"
 	merkledag "github.com/ipfs/go-ipfs/merkledag"
 	mdtest "github.com/ipfs/go-ipfs/merkledag/test"
 	ft "github.com/ipfs/go-ipfs/unixfs"
 	uio "github.com/ipfs/go-ipfs/unixfs/io"
+	chunk "gx/ipfs/QmQXcAyrC4VBu9ZBqxoCthPot9PNhb4Uiw6iBDfQXudZJd/go-ipfs-chunker"
 
 	u "gx/ipfs/QmNiJuT8Ja3hMVpBHXv3Q6dwmperaQ6JjLtpMQgMCD7xvx/go-ipfs-util"
 	ipld "gx/ipfs/Qme5bWv7wtjUNGsK2BNGVUFPKiuxWrsqrtvYwCLRw8YFES/go-ipld-format"

merkledag/merkledag_test.go

@@ -17,13 +17,13 @@ import (
 	bstest "github.com/ipfs/go-ipfs/blockservice/test"
 	offline "github.com/ipfs/go-ipfs/exchange/offline"
 	imp "github.com/ipfs/go-ipfs/importer"
-	chunk "github.com/ipfs/go-ipfs/importer/chunk"
 	. "github.com/ipfs/go-ipfs/merkledag"
 	mdpb "github.com/ipfs/go-ipfs/merkledag/pb"
 	dstest "github.com/ipfs/go-ipfs/merkledag/test"
 	uio "github.com/ipfs/go-ipfs/unixfs/io"
 
 	u "gx/ipfs/QmNiJuT8Ja3hMVpBHXv3Q6dwmperaQ6JjLtpMQgMCD7xvx/go-ipfs-util"
+	chunk "gx/ipfs/QmQXcAyrC4VBu9ZBqxoCthPot9PNhb4Uiw6iBDfQXudZJd/go-ipfs-chunker"
 	cid "gx/ipfs/QmcZfnkapfECQGcLZaf9B79NRg7cRa9EnZh4LSbkCzwNvY/go-cid"
 	ipld "gx/ipfs/Qme5bWv7wtjUNGsK2BNGVUFPKiuxWrsqrtvYwCLRw8YFES/go-ipld-format"
 	blocks "gx/ipfs/Qmej7nf81hi2x2tvjRBF3mcp74sQyuDH4VMYDGd1YtXjb2/go-block-format"

mfs/file.go

@@ -5,10 +5,10 @@ import (
 	"fmt"
 	"sync"
 
-	chunk "github.com/ipfs/go-ipfs/importer/chunk"
 	dag "github.com/ipfs/go-ipfs/merkledag"
 	ft "github.com/ipfs/go-ipfs/unixfs"
 	mod "github.com/ipfs/go-ipfs/unixfs/mod"
+	chunk "gx/ipfs/QmQXcAyrC4VBu9ZBqxoCthPot9PNhb4Uiw6iBDfQXudZJd/go-ipfs-chunker"
 
 	ipld "gx/ipfs/Qme5bWv7wtjUNGsK2BNGVUFPKiuxWrsqrtvYwCLRw8YFES/go-ipld-format"
 )

mfs/mfs_test.go

@@ -18,11 +18,11 @@ import (
 	bserv "github.com/ipfs/go-ipfs/blockservice"
 	offline "github.com/ipfs/go-ipfs/exchange/offline"
 	importer "github.com/ipfs/go-ipfs/importer"
-	chunk "github.com/ipfs/go-ipfs/importer/chunk"
 	dag "github.com/ipfs/go-ipfs/merkledag"
 	"github.com/ipfs/go-ipfs/path"
 	ft "github.com/ipfs/go-ipfs/unixfs"
 	uio "github.com/ipfs/go-ipfs/unixfs/io"
+	chunk "gx/ipfs/QmQXcAyrC4VBu9ZBqxoCthPot9PNhb4Uiw6iBDfQXudZJd/go-ipfs-chunker"
 
 	u "gx/ipfs/QmNiJuT8Ja3hMVpBHXv3Q6dwmperaQ6JjLtpMQgMCD7xvx/go-ipfs-util"
 	ds "gx/ipfs/QmPpegoMqhAEqjncrzArm7KVWAkCm78rqL2DPuNjhPrshg/go-datastore"

package.json

@@ -521,6 +521,11 @@
       "hash": "Qmb3jLEFAQrqdVgWUajqEyuuDoavkSq1XQXz6tWdFWF995",
       "name": "go-ipfs-posinfo",
       "version": "0.0.1"
+    },
+      "author": "hsanjuan",
+      "hash": "QmQXcAyrC4VBu9ZBqxoCthPot9PNhb4Uiw6iBDfQXudZJd",
+      "name": "go-ipfs-chunker",
+      "version": "0.0.1"
     }
   ],
   "gxVersion": "0.10.0",

tar/format.go

@@ -9,11 +9,11 @@ import (
 	"strings"
 
 	importer "github.com/ipfs/go-ipfs/importer"
-	chunk "github.com/ipfs/go-ipfs/importer/chunk"
 	dag "github.com/ipfs/go-ipfs/merkledag"
 	dagutil "github.com/ipfs/go-ipfs/merkledag/utils"
 	path "github.com/ipfs/go-ipfs/path"
 	uio "github.com/ipfs/go-ipfs/unixfs/io"
+	chunk "gx/ipfs/QmQXcAyrC4VBu9ZBqxoCthPot9PNhb4Uiw6iBDfQXudZJd/go-ipfs-chunker"
 
 	logging "gx/ipfs/QmRb5jh8z2E8hMGN2tkvs1yHynUanqnZ3UeKwgN1i9P1F8/go-log"
 	ipld "gx/ipfs/Qme5bWv7wtjUNGsK2BNGVUFPKiuxWrsqrtvYwCLRw8YFES/go-ipld-format"

unixfs/mod/dagmodifier.go

@@ -8,12 +8,12 @@ import (
 	"errors"
 	"io"
 
-	chunk "github.com/ipfs/go-ipfs/importer/chunk"
 	help "github.com/ipfs/go-ipfs/importer/helpers"
 	trickle "github.com/ipfs/go-ipfs/importer/trickle"
 	mdag "github.com/ipfs/go-ipfs/merkledag"
 	ft "github.com/ipfs/go-ipfs/unixfs"
 	uio "github.com/ipfs/go-ipfs/unixfs/io"
+	chunk "gx/ipfs/QmQXcAyrC4VBu9ZBqxoCthPot9PNhb4Uiw6iBDfQXudZJd/go-ipfs-chunker"
 
 	proto "gx/ipfs/QmZ4Qi3GaRbjcx28Sme5eMH7RQjGkt8wHxt2a65oLaeFEV/gogo-protobuf/proto"
 	cid "gx/ipfs/QmcZfnkapfECQGcLZaf9B79NRg7cRa9EnZh4LSbkCzwNvY/go-cid"

unixfs/test/utils.go

@@ -8,12 +8,12 @@ import (
 	"io/ioutil"
 	"testing"
 
-	"github.com/ipfs/go-ipfs/importer/chunk"
 	h "github.com/ipfs/go-ipfs/importer/helpers"
 	trickle "github.com/ipfs/go-ipfs/importer/trickle"
 	mdag "github.com/ipfs/go-ipfs/merkledag"
 	mdagmock "github.com/ipfs/go-ipfs/merkledag/test"
 	ft "github.com/ipfs/go-ipfs/unixfs"
+	"gx/ipfs/QmQXcAyrC4VBu9ZBqxoCthPot9PNhb4Uiw6iBDfQXudZJd/go-ipfs-chunker"
 
 	u "gx/ipfs/QmNiJuT8Ja3hMVpBHXv3Q6dwmperaQ6JjLtpMQgMCD7xvx/go-ipfs-util"
 	mh "gx/ipfs/QmZyZDi491cCNTLfAhwcaDii2Kg4pwKRkhqQzURGDvY6ua/go-multihash"