opensource/kubo
1
0
Fork 0
mirror of https://github.com/ipfs/kubo.git synced 2025-06-26 23:53:19 +08:00
Code Issues Projects Releases Wiki Activity

coreapi unixfs: pin/local/hash-only options

Browse Source 
License: MIT
Signed-off-by: Łukasz Magiera <magik6k@gmail.com>
This commit is contained in:
Łukasz Magiera
2018-09-20 23:05:22 +02:00
parent c056e5a8de
commit 49946c69c4
3 changed files with 138 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
core/coreapi/interface/options/unixfs.go
View File

@ -21,6 +21,10 @@ type UnixfsAddSettings struct {
Chunker string
Layout Layout
Pin bool
OnlyHash bool
Local bool
}
type UnixfsAddOption func(*UnixfsAddSettings) error
@ -36,6 +40,10 @@ func UnixfsAddOptions(opts ...UnixfsAddOption) (*UnixfsAddSettings, error) {
Chunker: "size-262144",
Layout: BalancedLayout,
Pin: false,
OnlyHash: false,
Local: false,
}
for _, opt := range opts {
@ -94,3 +102,24 @@ func (unixfsOpts) Layout(layout Layout) UnixfsAddOption {
return nil
}
}
func (unixfsOpts) Pin(pin bool) UnixfsAddOption {
return func(settings *UnixfsAddSettings) error {
settings.Pin = pin
return nil
}
}
func (unixfsOpts) HashOnly(hashOnly bool) UnixfsAddOption {
return func(settings *UnixfsAddSettings) error {
settings.OnlyHash = hashOnly
return nil
}
}
func (unixfsOpts) Local(local bool) UnixfsAddOption {
return func(settings *UnixfsAddSettings) error {
settings.Local = local
return nil
}
}

57
core/coreapi/unixfs.go
View File

@ -4,18 +4,25 @@ import (
"context"
"errors"
"fmt"
"github.com/ipfs/go-ipfs/core"
"io"
coreiface "github.com/ipfs/go-ipfs/core/coreapi/interface"
"github.com/ipfs/go-ipfs/core/coreapi/interface/options"
"github.com/ipfs/go-ipfs/core/coreunix"
uio "gx/ipfs/QmPL8bYtbACcSFFiSr4s2du7Na382NxRADR8hC7D9FkEA2/go-unixfs/io"
mh "gx/ipfs/QmPnFwZ2JXKnXgMw8CdBPxn7FWh6LLdjUjxV1fKHuJnkr8/go-multihash"
cidutil "gx/ipfs/QmQJSeE3CX4zos9qeaG8EhecEK9zvrTEfTG84J8C5NVRwt/go-cidutil"
offline "gx/ipfs/QmR5miWuikPxWyUrzMYJVmFUcD44pGdtc98h9Qsbp4YcJw/go-ipfs-exchange-offline"
"gx/ipfs/QmSP88ryZkHSRn1fnngAaV2Vcn63WUJzAavnRM9CVdU1Ky/go-ipfs-cmdkit/files"
ft "gx/ipfs/QmU4x3742bvgfxJsByEDpBnifJqjJdV6x528co4hwKCn46/go-unixfs"
uio "gx/ipfs/QmU4x3742bvgfxJsByEDpBnifJqjJdV6x528co4hwKCn46/go-unixfs/io"
mfs "gx/ipfs/QmahrY1adY4wvtYEtoGjpZ2GUohTyukrkMkwUR9ytRjTG2/go-mfs"
dag "gx/ipfs/QmcBoNcAP6qDjgRBew7yjvCqHq7p5jMstE44jPUBWBxzsV/go-merkledag"
dagtest "gx/ipfs/QmcBoNcAP6qDjgRBew7yjvCqHq7p5jMstE44jPUBWBxzsV/go-merkledag/test"
blockservice "gx/ipfs/QmcRecCZWM2NZfCQrCe97Ch3Givv8KKEP82tGUDntzdLFe/go-blockservice"
ipld "gx/ipfs/QmdDXJs4axxefSPgK6Y1QhpJWKuDPnGJiqgq4uncb4rFHL/go-ipld-format"
bstore "gx/ipfs/QmdriVJgKx4JADRgh3cYPXqXmsa1A45SvFki1nDWHhQNtC/go-ipfs-blockstore"
)
type UnixfsAPI CoreAPI
@ -59,7 +66,33 @@ func (api *UnixfsAPI) Add(ctx context.Context, r io.ReadCloser, opts ...options.
prefix.MhType = settings.MhType
prefix.MhLength = -1
fileAdder, err := coreunix.NewAdder(ctx, api.node.Pinning, api.node.Blockstore, api.node.DAG)
n := api.node
if settings.OnlyHash {
nilnode, err := core.NewNode(ctx, &core.BuildCfg{
//TODO: need this to be true or all files
// hashed will be stored in memory!
NilRepo: true,
})
if err != nil {
return nil, err
}
n = nilnode
}
addblockstore := n.Blockstore
//if !(fscache || nocopy) {
addblockstore = bstore.NewGCBlockstore(n.BaseBlocks, n.GCLocker)
//}
exch := n.Exchange
if settings.Local {
exch = offline.Exchange(addblockstore)
}
bserv := blockservice.New(addblockstore, exch) // hash security 001
dserv := dag.NewDAGService(bserv)
fileAdder, err := coreunix.NewAdder(ctx, n.Pinning, n.Blockstore, dserv)
if err != nil {
return nil, err
}
@ -68,7 +101,7 @@ func (api *UnixfsAPI) Add(ctx context.Context, r io.ReadCloser, opts ...options.
//fileAdder.Progress = progress
//fileAdder.Hidden = hidden
//fileAdder.Wrap = wrap
//fileAdder.Pin = dopin
fileAdder.Pin = settings.Pin
fileAdder.Silent = true
fileAdder.RawLeaves = settings.RawLeaves
//fileAdder.NoCopy = nocopy
@ -91,6 +124,19 @@ func (api *UnixfsAPI) Add(ctx context.Context, r io.ReadCloser, opts ...options.
}
}
if settings.OnlyHash {
md := dagtest.Mock()
emptyDirNode := ft.EmptyDirNode()
// Use the same prefix for the "empty" MFS root as for the file adder.
emptyDirNode.SetCidBuilder(fileAdder.CidBuilder)
mr, err := mfs.NewRoot(ctx, md, emptyDirNode, nil)
if err != nil {
return nil, err
}
fileAdder.SetMfsRoot(mr)
}
err = fileAdder.AddFile(files.NewReaderFile("", "", r, nil))
if err != nil {
return nil, err
@ -101,8 +147,11 @@ func (api *UnixfsAPI) Add(ctx context.Context, r io.ReadCloser, opts ...options.
return nil, err
}
return coreiface.IpfsPath(nd.Cid()), err
if settings.Pin {
err = fileAdder.PinRoot()
}
return coreiface.IpfsPath(nd.Cid()), err
}
// Cat returns the data contained by an IPFS or IPNS object(s) at path `p`.

56
core/coreapi/unixfs_test.go
View File

@ -204,6 +204,13 @@ func TestAdd(t *testing.T) {
path: "/ipfs/QmNNhDGttafX3M1wKWixGre6PrLFGjnoPEDXjBYpTv93HP",
opts: []options.UnixfsAddOption{options.Unixfs.Chunker("size-4"), options.Unixfs.Layout(options.TrickleLeyout)},
},
// Local
{
name: "addLocal", // better cases in sharness
data: helloStr,
path: hello,
opts: []options.UnixfsAddOption{options.Unixfs.Local(true)},
},
}
for _, testCase := range cases {
@ -244,6 +251,55 @@ func TestAdd(t *testing.T) {
}
}
func TestAddPinned(t *testing.T) {
ctx := context.Background()
_, api, err := makeAPI(ctx)
if err != nil {
t.Error(err)
}
str := strings.NewReader(helloStr)
_, err = api.Unixfs().Add(ctx, ioutil.NopCloser(str), options.Unixfs.Pin(true))
if err != nil {
t.Error(err)
}
pins, err := api.Pin().Ls(ctx)
if len(pins) != 1 {
t.Fatalf("expected 1 pin, got %d", len(pins))
}
if pins[0].Path().String() != "/ipld/QmQy2Dw4Wk7rdJKjThjYXzfFJNaRKRHhHP5gHHXroJMYxk" {
t.Fatalf("got unexpected pin: %s", pins[0].Path().String())
}
}
func TestAddHashOnly(t *testing.T) {
ctx := context.Background()
_, api, err := makeAPI(ctx)
if err != nil {
t.Error(err)
}
str := strings.NewReader(helloStr)
p, err := api.Unixfs().Add(ctx, ioutil.NopCloser(str), options.Unixfs.HashOnly(true))
if err != nil {
t.Error(err)
}
if p.String() != hello {
t.Errorf("unxepected path: %s", p.String())
}
_, err = api.Block().Get(ctx, p)
if err == nil {
t.Fatal("expected an error")
}
if err.Error() != "blockservice: key not found" {
t.Errorf("unxepected error: %s", err.Error())
}
}
func TestCatEmptyFile(t *testing.T) {
ctx := context.Background()
node, api, err := makeAPI(ctx)