opensource/kubo
1
0
Fork 0
mirror of https://github.com/ipfs/kubo.git synced 2025-06-30 01:52:26 +08:00
Code Issues Projects Releases Wiki Activity

Enable CidV1 (and other prefixes) in the Dag Modifier.

Browse Source 
License: MIT
Signed-off-by: Kevin Atkinson <k@kevina.org>
This commit is contained in:
Kevin Atkinson
2017-08-08 18:19:46 -04:00
parent a3bd2c23b3
commit 2e15dcb647
6 changed files with 178 additions and 165 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
importer/trickle/trickledag.go
View File

@ -9,6 +9,7 @@ import (
dag "github.com/ipfs/go-ipfs/merkledag"
ft "github.com/ipfs/go-ipfs/unixfs"
cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid"
node "gx/ipfs/QmPN7cwmpcc4DWXb4KTB9dNAJgjuPY69h3npsMfhRrQL9c/go-ipld-format"
)
@ -239,6 +240,7 @@ type VerifyParams struct {
Getter node.NodeGetter
Direct int
LayerRepeat int
Prefix *cid.Prefix
RawLeaves bool
}
@ -250,6 +252,7 @@ func VerifyTrickleDagStructure(nd node.Node, p VerifyParams) error {
// Recursive call for verifying the structure of a trickledag
func verifyTDagRec(n node.Node, depth int, p VerifyParams) error {
codec := cid.DagProtobuf
if depth == 0 {
if len(n.Links()) > 0 {
return errors.New("expected direct block")
@ -269,19 +272,36 @@ func verifyTDagRec(n node.Node, depth int, p VerifyParams) error {
if p.RawLeaves {
return errors.New("expected raw leaf, got a protobuf node")
}
return nil
case *dag.RawNode:
if !p.RawLeaves {
return errors.New("expected protobuf node as leaf")
}
return nil
codec = cid.Raw
default:
return errors.New("expected ProtoNode or RawNode")
}
}
// verify prefix
if p.Prefix != nil {
prefix := n.Cid().Prefix()
expect := *p.Prefix // make a copy
expect.Codec = uint64(codec)
if codec == cid.Raw && expect.Version == 0 {
expect.Version = 1
}
if expect.MhLength == -1 {
expect.MhLength = prefix.MhLength
}
if prefix != expect {
return fmt.Errorf("unexpected cid prefix: expected: %v; got %v", expect, prefix)
}
}
if depth == 0 {
return nil
}
nd, ok := n.(*dag.ProtoNode)
if !ok {
return errors.New("expected ProtoNode")

3
merkledag/node.go
View File

@ -42,6 +42,9 @@ var v1CidPrefix = cid.Prefix{
Version: 1,
}
func V0CidPrefix() cid.Prefix { return v0CidPrefix }
func V1CidPrefix() cid.Prefix { return v1CidPrefix }
// PrefixForCidVersion returns the Protobuf prefix for a given CID version
func PrefixForCidVersion(version int) (cid.Prefix, error) {
switch version {

12
unixfs/io/dagreader_test.go
View File

@ -17,7 +17,7 @@ import (
func TestBasicRead(t *testing.T) {
dserv := testu.GetDAGServ()
inbuf, node := testu.GetRandomNode(t, dserv, 1024, testu.ProtoBufLeaves)
inbuf, node := testu.GetRandomNode(t, dserv, 1024, testu.UseProtoBufLeaves)
ctx, closer := context.WithCancel(context.Background())
defer closer()
@ -44,7 +44,7 @@ func TestSeekAndRead(t *testing.T) {
inbuf[i] = byte(i)
}
node := testu.GetNode(t, dserv, inbuf, testu.ProtoBufLeaves)
node := testu.GetNode(t, dserv, inbuf, testu.UseProtoBufLeaves)
ctx, closer := context.WithCancel(context.Background())
defer closer()
@ -84,7 +84,7 @@ func TestRelativeSeek(t *testing.T) {
}
inbuf[1023] = 1 // force the reader to be 1024 bytes
node := testu.GetNode(t, dserv, inbuf, testu.ProtoBufLeaves)
node := testu.GetNode(t, dserv, inbuf, testu.UseProtoBufLeaves)
reader, err := NewDagReader(ctx, node, dserv)
if err != nil {
@ -160,7 +160,7 @@ func TestBadPBData(t *testing.T) {
func TestMetadataNode(t *testing.T) {
dserv := testu.GetDAGServ()
rdata, rnode := testu.GetRandomNode(t, dserv, 512, testu.ProtoBufLeaves)
rdata, rnode := testu.GetRandomNode(t, dserv, 512, testu.UseProtoBufLeaves)
_, err := dserv.Add(rnode)
if err != nil {
t.Fatal(err)
@ -203,7 +203,7 @@ func TestMetadataNode(t *testing.T) {
func TestWriteTo(t *testing.T) {
dserv := testu.GetDAGServ()
inbuf, node := testu.GetRandomNode(t, dserv, 1024, testu.ProtoBufLeaves)
inbuf, node := testu.GetRandomNode(t, dserv, 1024, testu.UseProtoBufLeaves)
ctx, closer := context.WithCancel(context.Background())
defer closer()
@ -225,7 +225,7 @@ func TestWriteTo(t *testing.T) {
func TestReaderSzie(t *testing.T) {
dserv := testu.GetDAGServ()
size := int64(1024)
_, node := testu.GetRandomNode(t, dserv, size, testu.ProtoBufLeaves)
_, node := testu.GetRandomNode(t, dserv, size, testu.UseProtoBufLeaves)
ctx, closer := context.WithCancel(context.Background())
defer closer()

24
unixfs/mod/dagmodifier.go
View File

@ -40,6 +40,7 @@ type DagModifier struct {
curWrOff uint64
wrBuf *bytes.Buffer
Prefix cid.Prefix
RawLeaves bool
read uio.DagReader
@ -47,6 +48,10 @@ type DagModifier struct {
var ErrNotUnixfs = fmt.Errorf("dagmodifier only supports unixfs nodes (proto or raw)")
// NewDagModifier returns a new DagModifier, the Cid prefix for newly
// created nodes will be inherted from the passed in node. If the Cid
// version if not 0 raw leaves will also be enabled. The Prefix and
// RawLeaves options can be overridden by changing them after the call.
func NewDagModifier(ctx context.Context, from node.Node, serv mdag.DAGService, spl chunk.SplitterGen) (*DagModifier, error) {
switch from.(type) {
case *mdag.ProtoNode, *mdag.RawNode:
@ -55,11 +60,20 @@ func NewDagModifier(ctx context.Context, from node.Node, serv mdag.DAGService, s
return nil, ErrNotUnixfs
}
prefix := from.Cid().Prefix()
prefix.Codec = cid.DagProtobuf
rawLeaves := false
if prefix.Version > 0 {
rawLeaves = true
}
return &DagModifier{
curNode: from.Copy(),
dagserv: serv,
splitter: spl,
ctx: ctx,
curNode: from.Copy(),
dagserv: serv,
splitter: spl,
ctx: ctx,
Prefix: prefix,
RawLeaves: rawLeaves,
}, nil
}
@ -240,6 +254,7 @@ func (dm *DagModifier) modifyDag(n node.Node, offset uint64, data io.Reader) (*c
nd := new(mdag.ProtoNode)
nd.SetData(b)
nd.SetPrefix(&nd0.Prefix)
k, err := dm.dagserv.Add(nd)
if err != nil {
return nil, false, err
@ -345,6 +360,7 @@ func (dm *DagModifier) appendData(nd node.Node, spl chunk.Splitter) (node.Node,
dbp := &help.DagBuilderParams{
Dagserv: dm.dagserv,
Maxlinks: help.DefaultLinksPerBlock,
Prefix: &dm.Prefix,
RawLeaves: dm.RawLeaves,
}
return trickle.TrickleAppend(dm.ctx, nd, dbp.New(spl))

247
unixfs/mod/dagmodifier_test.go
View File

@ -16,7 +16,7 @@ import (
u "gx/ipfs/QmSU6eubNdhXjFBJBSksTp8kv8YRub8mGAPv8tVJHmL2EU/go-ipfs-util"
)
func testModWrite(t *testing.T, beg, size uint64, orig []byte, dm *DagModifier, rawLeaves testu.UseRawLeaves) []byte {
func testModWrite(t *testing.T, beg, size uint64, orig []byte, dm *DagModifier, opts testu.NodeOpts) []byte {
newdata := make([]byte, size)
r := u.NewTimeSeededRand()
r.Read(newdata)
@ -35,6 +35,12 @@ func testModWrite(t *testing.T, beg, size uint64, orig []byte, dm *DagModifier,
t.Fatalf("Mod length not correct! %d != %d", nmod, size)
}
verifyNode(t, orig, dm, opts)
return orig
}
func verifyNode(t *testing.T, orig []byte, dm *DagModifier, opts testu.NodeOpts) {
nd, err := dm.GetNode()
if err != nil {
t.Fatal(err)
@ -44,10 +50,11 @@ func testModWrite(t *testing.T, beg, size uint64, orig []byte, dm *DagModifier,
Getter: dm.dagserv,
Direct: h.DefaultLinksPerBlock,
LayerRepeat: 4,
RawLeaves: bool(rawLeaves),
Prefix: &opts.Prefix,
RawLeaves: opts.RawLeavesUsed,
})
if err != nil {
t.Error(err)
t.Fatal(err)
}
rd, err := uio.NewDagReader(context.Background(), nd, dm.dagserv)
@ -64,20 +71,20 @@ func testModWrite(t *testing.T, beg, size uint64, orig []byte, dm *DagModifier,
if err != nil {
t.Fatal(err)
}
return orig
}
func runBothSubtests(t *testing.T, tfunc func(*testing.T, testu.UseRawLeaves)) {
t.Run("leaves=ProtoBuf", func(t *testing.T) { tfunc(t, testu.ProtoBufLeaves) })
t.Run("leaves=Raw", func(t *testing.T) { tfunc(t, testu.RawLeaves) })
func runAllSubtests(t *testing.T, tfunc func(*testing.T, testu.NodeOpts)) {
t.Run("opts=ProtoBufLeaves", func(t *testing.T) { tfunc(t, testu.UseProtoBufLeaves) })
t.Run("opts=RawLeaves", func(t *testing.T) { tfunc(t, testu.UseRawLeaves) })
t.Run("opts=CidV1", func(t *testing.T) { tfunc(t, testu.UseCidV1) })
}
func TestDagModifierBasic(t *testing.T) {
runBothSubtests(t, testDagModifierBasic)
runAllSubtests(t, testDagModifierBasic)
}
func testDagModifierBasic(t *testing.T, rawLeaves testu.UseRawLeaves) {
func testDagModifierBasic(t *testing.T, opts testu.NodeOpts) {
dserv := testu.GetDAGServ()
b, n := testu.GetRandomNode(t, dserv, 50000, rawLeaves)
b, n := testu.GetRandomNode(t, dserv, 50000, opts)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
@ -85,33 +92,35 @@ func testDagModifierBasic(t *testing.T, rawLeaves testu.UseRawLeaves) {
if err != nil {
t.Fatal(err)
}
dagmod.RawLeaves = bool(rawLeaves)
if opts.ForceRawLeaves {
dagmod.RawLeaves = true
}
// Within zero block
beg := uint64(15)
length := uint64(60)
t.Log("Testing mod within zero block")
b = testModWrite(t, beg, length, b, dagmod, rawLeaves)
b = testModWrite(t, beg, length, b, dagmod, opts)
// Within bounds of existing file
beg = 1000
length = 4000
t.Log("Testing mod within bounds of existing multiblock file.")
b = testModWrite(t, beg, length, b, dagmod, rawLeaves)
b = testModWrite(t, beg, length, b, dagmod, opts)
// Extend bounds
beg = 49500
length = 4000
t.Log("Testing mod that extends file.")
b = testModWrite(t, beg, length, b, dagmod, rawLeaves)
b = testModWrite(t, beg, length, b, dagmod, opts)
// "Append"
beg = uint64(len(b))
length = 3000
t.Log("Testing pure append")
_ = testModWrite(t, beg, length, b, dagmod, rawLeaves)
_ = testModWrite(t, beg, length, b, dagmod, opts)
// Verify reported length
node, err := dagmod.GetNode()
@ -131,11 +140,11 @@ func testDagModifierBasic(t *testing.T, rawLeaves testu.UseRawLeaves) {
}
func TestMultiWrite(t *testing.T) {
runBothSubtests(t, testMultiWrite)
runAllSubtests(t, testMultiWrite)
}
func testMultiWrite(t *testing.T, rawLeaves testu.UseRawLeaves) {
func testMultiWrite(t *testing.T, opts testu.NodeOpts) {
dserv := testu.GetDAGServ()
n := testu.GetEmptyNode(t, dserv, rawLeaves)
n := testu.GetEmptyNode(t, dserv, opts)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
@ -144,7 +153,9 @@ func testMultiWrite(t *testing.T, rawLeaves testu.UseRawLeaves) {
if err != nil {
t.Fatal(err)
}
dagmod.RawLeaves = bool(rawLeaves)
if opts.ForceRawLeaves {
dagmod.RawLeaves = true
}
data := make([]byte, 4000)
u.NewTimeSeededRand().Read(data)
@ -167,32 +178,16 @@ func testMultiWrite(t *testing.T, rawLeaves testu.UseRawLeaves) {
t.Fatal("Size was reported incorrectly")
}
}
nd, err := dagmod.GetNode()
if err != nil {
t.Fatal(err)
}
read, err := uio.NewDagReader(context.Background(), nd, dserv)
if err != nil {
t.Fatal(err)
}
rbuf, err := ioutil.ReadAll(read)
if err != nil {
t.Fatal(err)
}
err = testu.ArrComp(rbuf, data)
if err != nil {
t.Fatal(err)
}
verifyNode(t, data, dagmod, opts)
}
func TestMultiWriteAndFlush(t *testing.T) {
runBothSubtests(t, testMultiWriteAndFlush)
runAllSubtests(t, testMultiWriteAndFlush)
}
func testMultiWriteAndFlush(t *testing.T, rawLeaves testu.UseRawLeaves) {
func testMultiWriteAndFlush(t *testing.T, opts testu.NodeOpts) {
dserv := testu.GetDAGServ()
n := testu.GetEmptyNode(t, dserv, rawLeaves)
n := testu.GetEmptyNode(t, dserv, opts)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
@ -201,7 +196,9 @@ func testMultiWriteAndFlush(t *testing.T, rawLeaves testu.UseRawLeaves) {
if err != nil {
t.Fatal(err)
}
dagmod.RawLeaves = bool(rawLeaves)
if opts.ForceRawLeaves {
dagmod.RawLeaves = true
}
data := make([]byte, 20)
u.NewTimeSeededRand().Read(data)
@ -219,32 +216,16 @@ func testMultiWriteAndFlush(t *testing.T, rawLeaves testu.UseRawLeaves) {
t.Fatal(err)
}
}
nd, err := dagmod.GetNode()
if err != nil {
t.Fatal(err)
}
read, err := uio.NewDagReader(context.Background(), nd, dserv)
if err != nil {
t.Fatal(err)
}
rbuf, err := ioutil.ReadAll(read)
if err != nil {
t.Fatal(err)
}
err = testu.ArrComp(rbuf, data)
if err != nil {
t.Fatal(err)
}
verifyNode(t, data, dagmod, opts)
}
func TestWriteNewFile(t *testing.T) {
runBothSubtests(t, testWriteNewFile)
runAllSubtests(t, testWriteNewFile)
}
func testWriteNewFile(t *testing.T, rawLeaves testu.UseRawLeaves) {
func testWriteNewFile(t *testing.T, opts testu.NodeOpts) {
dserv := testu.GetDAGServ()
n := testu.GetEmptyNode(t, dserv, rawLeaves)
n := testu.GetEmptyNode(t, dserv, opts)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
@ -253,7 +234,9 @@ func testWriteNewFile(t *testing.T, rawLeaves testu.UseRawLeaves) {
if err != nil {
t.Fatal(err)
}
dagmod.RawLeaves = bool(rawLeaves)
if opts.ForceRawLeaves {
dagmod.RawLeaves = true
}
towrite := make([]byte, 2000)
u.NewTimeSeededRand().Read(towrite)
@ -266,32 +249,15 @@ func testWriteNewFile(t *testing.T, rawLeaves testu.UseRawLeaves) {
t.Fatal("Wrote wrong amount")
}
nd, err := dagmod.GetNode()
if err != nil {
t.Fatal(err)
}
read, err := uio.NewDagReader(ctx, nd, dserv)
if err != nil {
t.Fatal(err)
}
data, err := ioutil.ReadAll(read)
if err != nil {
t.Fatal(err)
}
if err := testu.ArrComp(data, towrite); err != nil {
t.Fatal(err)
}
verifyNode(t, towrite, dagmod, opts)
}
func TestMultiWriteCoal(t *testing.T) {
runBothSubtests(t, testMultiWriteCoal)
runAllSubtests(t, testMultiWriteCoal)
}
func testMultiWriteCoal(t *testing.T, rawLeaves testu.UseRawLeaves) {
func testMultiWriteCoal(t *testing.T, opts testu.NodeOpts) {
dserv := testu.GetDAGServ()
n := testu.GetEmptyNode(t, dserv, rawLeaves)
n := testu.GetEmptyNode(t, dserv, opts)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
@ -300,7 +266,9 @@ func testMultiWriteCoal(t *testing.T, rawLeaves testu.UseRawLeaves) {
if err != nil {
t.Fatal(err)
}
dagmod.RawLeaves = bool(rawLeaves)
if opts.ForceRawLeaves {
dagmod.RawLeaves = true
}
data := make([]byte, 1000)
u.NewTimeSeededRand().Read(data)
@ -316,34 +284,16 @@ func testMultiWriteCoal(t *testing.T, rawLeaves testu.UseRawLeaves) {
}
}
nd, err := dagmod.GetNode()
if err != nil {
t.Fatal(err)
}
read, err := uio.NewDagReader(context.Background(), nd, dserv)
if err != nil {
t.Fatal(err)
}
dagmod.RawLeaves = bool(rawLeaves)
rbuf, err := ioutil.ReadAll(read)
if err != nil {
t.Fatal(err)
}
err = testu.ArrComp(rbuf, data)
if err != nil {
t.Fatal(err)
}
verifyNode(t, data, dagmod, opts)
}
func TestLargeWriteChunks(t *testing.T) {
runBothSubtests(t, testLargeWriteChunks)
runAllSubtests(t, testLargeWriteChunks)
}
func testLargeWriteChunks(t *testing.T, rawLeaves testu.UseRawLeaves) {
func testLargeWriteChunks(t *testing.T, opts testu.NodeOpts) {
dserv := testu.GetDAGServ()
n := testu.GetEmptyNode(t, dserv, rawLeaves)
n := testu.GetEmptyNode(t, dserv, opts)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
@ -352,7 +302,9 @@ func testLargeWriteChunks(t *testing.T, rawLeaves testu.UseRawLeaves) {
if err != nil {
t.Fatal(err)
}
dagmod.RawLeaves = bool(rawLeaves)
if opts.ForceRawLeaves {
dagmod.RawLeaves = true
}
wrsize := 1000
datasize := 10000000
@ -378,15 +330,14 @@ func testLargeWriteChunks(t *testing.T, rawLeaves testu.UseRawLeaves) {
if err = testu.ArrComp(out, data); err != nil {
t.Fatal(err)
}
}
func TestDagTruncate(t *testing.T) {
runBothSubtests(t, testDagTruncate)
runAllSubtests(t, testDagTruncate)
}
func testDagTruncate(t *testing.T, rawLeaves testu.UseRawLeaves) {
func testDagTruncate(t *testing.T, opts testu.NodeOpts) {
dserv := testu.GetDAGServ()
b, n := testu.GetRandomNode(t, dserv, 50000, rawLeaves)
b, n := testu.GetRandomNode(t, dserv, 50000, opts)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
@ -394,7 +345,9 @@ func testDagTruncate(t *testing.T, rawLeaves testu.UseRawLeaves) {
if err != nil {
t.Fatal(err)
}
dagmod.RawLeaves = bool(rawLeaves)
if opts.ForceRawLeaves {
dagmod.RawLeaves = true
}
err = dagmod.Truncate(12345)
if err != nil {
@ -453,11 +406,11 @@ func testDagTruncate(t *testing.T, rawLeaves testu.UseRawLeaves) {
}
func TestSparseWrite(t *testing.T) {
runBothSubtests(t, testSparseWrite)
runAllSubtests(t, testSparseWrite)
}
func testSparseWrite(t *testing.T, rawLeaves testu.UseRawLeaves) {
func testSparseWrite(t *testing.T, opts testu.NodeOpts) {
dserv := testu.GetDAGServ()
n := testu.GetEmptyNode(t, dserv, rawLeaves)
n := testu.GetEmptyNode(t, dserv, opts)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
@ -465,7 +418,9 @@ func testSparseWrite(t *testing.T, rawLeaves testu.UseRawLeaves) {
if err != nil {
t.Fatal(err)
}
dagmod.RawLeaves = bool(rawLeaves)
if opts.ForceRawLeaves {
dagmod.RawLeaves = true
}
buf := make([]byte, 5000)
u.NewTimeSeededRand().Read(buf[2500:])
@ -495,11 +450,11 @@ func testSparseWrite(t *testing.T, rawLeaves testu.UseRawLeaves) {
}
func TestSeekPastEndWrite(t *testing.T) {
runBothSubtests(t, testSeekPastEndWrite)
runAllSubtests(t, testSeekPastEndWrite)
}
func testSeekPastEndWrite(t *testing.T, rawLeaves testu.UseRawLeaves) {
func testSeekPastEndWrite(t *testing.T, opts testu.NodeOpts) {
dserv := testu.GetDAGServ()
n := testu.GetEmptyNode(t, dserv, rawLeaves)
n := testu.GetEmptyNode(t, dserv, opts)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
@ -507,7 +462,9 @@ func testSeekPastEndWrite(t *testing.T, rawLeaves testu.UseRawLeaves) {
if err != nil {
t.Fatal(err)
}
dagmod.RawLeaves = bool(rawLeaves)
if opts.ForceRawLeaves {
dagmod.RawLeaves = true
}
buf := make([]byte, 5000)
u.NewTimeSeededRand().Read(buf[2500:])
@ -546,11 +503,11 @@ func testSeekPastEndWrite(t *testing.T, rawLeaves testu.UseRawLeaves) {
}
func TestRelativeSeek(t *testing.T) {
runBothSubtests(t, testRelativeSeek)
runAllSubtests(t, testRelativeSeek)
}
func testRelativeSeek(t *testing.T, rawLeaves testu.UseRawLeaves) {
func testRelativeSeek(t *testing.T, opts testu.NodeOpts) {
dserv := testu.GetDAGServ()
n := testu.GetEmptyNode(t, dserv, rawLeaves)
n := testu.GetEmptyNode(t, dserv, opts)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
@ -558,7 +515,9 @@ func testRelativeSeek(t *testing.T, rawLeaves testu.UseRawLeaves) {
if err != nil {
t.Fatal(err)
}
dagmod.RawLeaves = bool(rawLeaves)
if opts.ForceRawLeaves {
dagmod.RawLeaves = true
}
for i := 0; i < 64; i++ {
dagmod.Write([]byte{byte(i)})
@ -580,11 +539,11 @@ func testRelativeSeek(t *testing.T, rawLeaves testu.UseRawLeaves) {
}
func TestInvalidSeek(t *testing.T) {
runBothSubtests(t, testInvalidSeek)
runAllSubtests(t, testInvalidSeek)
}
func testInvalidSeek(t *testing.T, rawLeaves testu.UseRawLeaves) {
func testInvalidSeek(t *testing.T, opts testu.NodeOpts) {
dserv := testu.GetDAGServ()
n := testu.GetEmptyNode(t, dserv, rawLeaves)
n := testu.GetEmptyNode(t, dserv, opts)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
@ -592,7 +551,9 @@ func testInvalidSeek(t *testing.T, rawLeaves testu.UseRawLeaves) {
if err != nil {
t.Fatal(err)
}
dagmod.RawLeaves = bool(rawLeaves)
if opts.ForceRawLeaves {
dagmod.RawLeaves = true
}
_, err = dagmod.Seek(10, -10)
@ -602,12 +563,12 @@ func testInvalidSeek(t *testing.T, rawLeaves testu.UseRawLeaves) {
}
func TestEndSeek(t *testing.T) {
runBothSubtests(t, testEndSeek)
runAllSubtests(t, testEndSeek)
}
func testEndSeek(t *testing.T, rawLeaves testu.UseRawLeaves) {
func testEndSeek(t *testing.T, opts testu.NodeOpts) {
dserv := testu.GetDAGServ()
n := testu.GetEmptyNode(t, dserv, rawLeaves)
n := testu.GetEmptyNode(t, dserv, opts)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
@ -615,7 +576,9 @@ func testEndSeek(t *testing.T, rawLeaves testu.UseRawLeaves) {
if err != nil {
t.Fatal(err)
}
dagmod.RawLeaves = bool(rawLeaves)
if opts.ForceRawLeaves {
dagmod.RawLeaves = true
}
_, err = dagmod.Write(make([]byte, 100))
if err != nil {
@ -648,12 +611,12 @@ func testEndSeek(t *testing.T, rawLeaves testu.UseRawLeaves) {
}
func TestReadAndSeek(t *testing.T) {
runBothSubtests(t, testReadAndSeek)
runAllSubtests(t, testReadAndSeek)
}
func testReadAndSeek(t *testing.T, rawLeaves testu.UseRawLeaves) {
func testReadAndSeek(t *testing.T, opts testu.NodeOpts) {
dserv := testu.GetDAGServ()
n := testu.GetEmptyNode(t, dserv, rawLeaves)
n := testu.GetEmptyNode(t, dserv, opts)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
@ -661,7 +624,9 @@ func testReadAndSeek(t *testing.T, rawLeaves testu.UseRawLeaves) {
if err != nil {
t.Fatal(err)
}
dagmod.RawLeaves = bool(rawLeaves)
if opts.ForceRawLeaves {
dagmod.RawLeaves = true
}
writeBuf := []byte{0, 1, 2, 3, 4, 5, 6, 7}
dagmod.Write(writeBuf)
@ -720,12 +685,12 @@ func testReadAndSeek(t *testing.T, rawLeaves testu.UseRawLeaves) {
}
func TestCtxRead(t *testing.T) {
runBothSubtests(t, testCtxRead)
runAllSubtests(t, testCtxRead)
}
func testCtxRead(t *testing.T, rawLeaves testu.UseRawLeaves) {
func testCtxRead(t *testing.T, opts testu.NodeOpts) {
dserv := testu.GetDAGServ()
n := testu.GetEmptyNode(t, dserv, rawLeaves)
n := testu.GetEmptyNode(t, dserv, opts)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
@ -733,7 +698,9 @@ func testCtxRead(t *testing.T, rawLeaves testu.UseRawLeaves) {
if err != nil {
t.Fatal(err)
}
dagmod.RawLeaves = bool(rawLeaves)
if opts.ForceRawLeaves {
dagmod.RawLeaves = true
}
_, err = dagmod.Write([]byte{0, 1, 2, 3, 4, 5, 6, 7})
if err != nil {
@ -757,7 +724,7 @@ func testCtxRead(t *testing.T, rawLeaves testu.UseRawLeaves) {
func BenchmarkDagmodWrite(b *testing.B) {
b.StopTimer()
dserv := testu.GetDAGServ()
n := testu.GetEmptyNode(b, dserv, testu.ProtoBufLeaves)
n := testu.GetEmptyNode(b, dserv, testu.UseProtoBufLeaves)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()

29
unixfs/test/utils.go
View File

@ -15,6 +15,7 @@ import (
mdagmock "github.com/ipfs/go-ipfs/merkledag/test"
ft "github.com/ipfs/go-ipfs/unixfs"
cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid"
node "gx/ipfs/QmPN7cwmpcc4DWXb4KTB9dNAJgjuPY69h3npsMfhRrQL9c/go-ipld-format"
u "gx/ipfs/QmSU6eubNdhXjFBJBSksTp8kv8YRub8mGAPv8tVJHmL2EU/go-ipfs-util"
)
@ -29,20 +30,26 @@ func GetDAGServ() mdag.DAGService {
return mdagmock.Mock()
}
type UseRawLeaves bool
type NodeOpts struct {
Prefix cid.Prefix
// ForceRawLeaves if true will force the use of raw leaves
ForceRawLeaves bool
// RawLeavesUsed is true if raw leaves or either implicitly or explicitly enabled
RawLeavesUsed bool
}
const (
ProtoBufLeaves UseRawLeaves = false
RawLeaves UseRawLeaves = true
)
var UseProtoBufLeaves = NodeOpts{Prefix: mdag.V0CidPrefix()}
var UseRawLeaves = NodeOpts{Prefix: mdag.V0CidPrefix(), ForceRawLeaves: true, RawLeavesUsed: true}
var UseCidV1 = NodeOpts{Prefix: mdag.V1CidPrefix(), RawLeavesUsed: true}
func GetNode(t testing.TB, dserv mdag.DAGService, data []byte, rawLeaves UseRawLeaves) node.Node {
func GetNode(t testing.TB, dserv mdag.DAGService, data []byte, opts NodeOpts) node.Node {
in := bytes.NewReader(data)
dbp := h.DagBuilderParams{
Dagserv: dserv,
Maxlinks: h.DefaultLinksPerBlock,
RawLeaves: bool(rawLeaves),
Prefix: &opts.Prefix,
RawLeaves: opts.RawLeavesUsed,
}
node, err := trickle.TrickleLayout(dbp.New(SizeSplitterGen(500)(in)))
@ -53,18 +60,18 @@ func GetNode(t testing.TB, dserv mdag.DAGService, data []byte, rawLeaves UseRawL
return node
}
func GetEmptyNode(t testing.TB, dserv mdag.DAGService, rawLeaves UseRawLeaves) node.Node {
return GetNode(t, dserv, []byte{}, rawLeaves)
func GetEmptyNode(t testing.TB, dserv mdag.DAGService, opts NodeOpts) node.Node {
return GetNode(t, dserv, []byte{}, opts)
}
func GetRandomNode(t testing.TB, dserv mdag.DAGService, size int64, rawLeaves UseRawLeaves) ([]byte, node.Node) {
func GetRandomNode(t testing.TB, dserv mdag.DAGService, size int64, opts NodeOpts) ([]byte, node.Node) {
in := io.LimitReader(u.NewTimeSeededRand(), size)
buf, err := ioutil.ReadAll(in)
if err != nil {
t.Fatal(err)
}
node := GetNode(t, dserv, buf, rawLeaves)
node := GetNode(t, dserv, buf, opts)
return buf, node
}