opensource/kubo
1
0
Fork 0
mirror of https://github.com/ipfs/kubo.git synced 2025-06-28 00:39:31 +08:00
Code Issues Projects Releases Wiki Activity

Simplify code: use prefix instead of flag to determine if a url

Browse Source 
License: MIT
Signed-off-by: Kevin Atkinson <k@kevina.org>
This commit is contained in:
Kevin Atkinson
2018-06-22 21:39:15 -04:00
parent d59a6e99a6
commit 696a0f039e
11 changed files with 39 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
core/coreunix/add.go
View File

@ -19,12 +19,12 @@ import (
"github.com/ipfs/go-ipfs/pin"
unixfs "github.com/ipfs/go-ipfs/unixfs"
posinfo "gx/ipfs/QmUWsXLvYYDAaoAt9TPZpFX4ffHHMg46AHrz1ZLTN5ABbe/go-ipfs-posinfo"
ipld "gx/ipfs/QmWi2BYBL5gJ3CiAiQchg6rn1A8iBsrWy51EYxvHVjFvLb/go-ipld-format"
chunker "gx/ipfs/QmXnzH7wowyLZy8XJxxaQCVTgLMcDXdMBznmsrmQWCyiQV/go-ipfs-chunker"
cid "gx/ipfs/QmapdYm1b22Frv3k17fqrBYTFRxwiaVJkB299Mfn33edeB/go-cid"
logging "gx/ipfs/QmcVVHfdyv15GVPk7NrxdWjh2hLVccXnoD8j2tyQShiXJb/go-log"
files "gx/ipfs/QmdE4gMduCKCGAcczM2F5ioYDfdeKuPix138wrES1YSr7f/go-ipfs-cmdkit/files"
posinfo "gx/ipfs/QmdGSfmN4wWNXVs2XiwHbpjnUikJ7HyrTJNHyYGdodyJDC/go-ipfs-posinfo"
bstore "gx/ipfs/QmdpuJBPBZ6sLPj9BQpn3Rpi38BT2cF1QMiUfyzNWeySW4/go-ipfs-blockstore"
)

2
core/coreunix/add_test.go
View File

@ -18,9 +18,9 @@ import (
"github.com/ipfs/go-ipfs/repo/config"
blocks "gx/ipfs/QmTRCUvZLiir12Qr6MV3HKfKMHX8Nf1Vddn6t2g5nsQSb9/go-block-format"
pi "gx/ipfs/QmUWsXLvYYDAaoAt9TPZpFX4ffHHMg46AHrz1ZLTN5ABbe/go-ipfs-posinfo"
cid "gx/ipfs/QmapdYm1b22Frv3k17fqrBYTFRxwiaVJkB299Mfn33edeB/go-cid"
files "gx/ipfs/QmdE4gMduCKCGAcczM2F5ioYDfdeKuPix138wrES1YSr7f/go-ipfs-cmdkit/files"
pi "gx/ipfs/QmdGSfmN4wWNXVs2XiwHbpjnUikJ7HyrTJNHyYGdodyJDC/go-ipfs-posinfo"
blockstore "gx/ipfs/QmdpuJBPBZ6sLPj9BQpn3Rpi38BT2cF1QMiUfyzNWeySW4/go-ipfs-blockstore"
datastore "gx/ipfs/QmeiCcJfDW1GJnWUArudsv5rQsihpi4oyddPhdqo3CfX6i/go-datastore"
syncds "gx/ipfs/QmeiCcJfDW1GJnWUArudsv5rQsihpi4oyddPhdqo3CfX6i/go-datastore/sync"

2
filestore/filestore.go
View File

@ -11,9 +11,9 @@ import (
"context"
blocks "gx/ipfs/QmTRCUvZLiir12Qr6MV3HKfKMHX8Nf1Vddn6t2g5nsQSb9/go-block-format"
posinfo "gx/ipfs/QmUWsXLvYYDAaoAt9TPZpFX4ffHHMg46AHrz1ZLTN5ABbe/go-ipfs-posinfo"
cid "gx/ipfs/QmapdYm1b22Frv3k17fqrBYTFRxwiaVJkB299Mfn33edeB/go-cid"
logging "gx/ipfs/QmcVVHfdyv15GVPk7NrxdWjh2hLVccXnoD8j2tyQShiXJb/go-log"
posinfo "gx/ipfs/QmdGSfmN4wWNXVs2XiwHbpjnUikJ7HyrTJNHyYGdodyJDC/go-ipfs-posinfo"
blockstore "gx/ipfs/QmdpuJBPBZ6sLPj9BQpn3Rpi38BT2cF1QMiUfyzNWeySW4/go-ipfs-blockstore"
dsq "gx/ipfs/QmeiCcJfDW1GJnWUArudsv5rQsihpi4oyddPhdqo3CfX6i/go-datastore/query"
)

14
filestore/filestore_test.go
View File

@ -9,8 +9,8 @@ import (
dag "github.com/ipfs/go-ipfs/merkledag"
posinfo "gx/ipfs/QmUWsXLvYYDAaoAt9TPZpFX4ffHHMg46AHrz1ZLTN5ABbe/go-ipfs-posinfo"
cid "gx/ipfs/QmapdYm1b22Frv3k17fqrBYTFRxwiaVJkB299Mfn33edeB/go-cid"
posinfo "gx/ipfs/QmdGSfmN4wWNXVs2XiwHbpjnUikJ7HyrTJNHyYGdodyJDC/go-ipfs-posinfo"
blockstore "gx/ipfs/QmdpuJBPBZ6sLPj9BQpn3Rpi38BT2cF1QMiUfyzNWeySW4/go-ipfs-blockstore"
ds "gx/ipfs/QmeiCcJfDW1GJnWUArudsv5rQsihpi4oyddPhdqo3CfX6i/go-datastore"
)
@ -162,3 +162,15 @@ func TestDeletes(t *testing.T) {
}
}
}
func TestIsURL(t *testing.T) {
if !IsURL("http://www.example.com") {
t.Fatal("IsURL failed: http://www.example.com")
}
if !IsURL("https://www.example.com") {
t.Fatal("IsURL failed: https://www.example.com")
}
if IsURL("adir/afile") {
t.Fatal("IsURL recognized non-url")
}
}

13
filestore/fsrefstore.go
View File

@ -13,8 +13,8 @@ import (
dshelp "gx/ipfs/QmNP2u7bofwUQptHQGPfabGWtTCbxhNLSZKqbf1uzsup9V/go-ipfs-ds-help"
proto "gx/ipfs/QmT6n4mspWYEya864BhCUJEgyxiRfmiSY9ruQwTUNpRKaM/protobuf/proto"
blocks "gx/ipfs/QmTRCUvZLiir12Qr6MV3HKfKMHX8Nf1Vddn6t2g5nsQSb9/go-block-format"
posinfo "gx/ipfs/QmUWsXLvYYDAaoAt9TPZpFX4ffHHMg46AHrz1ZLTN5ABbe/go-ipfs-posinfo"
cid "gx/ipfs/QmapdYm1b22Frv3k17fqrBYTFRxwiaVJkB299Mfn33edeB/go-cid"
posinfo "gx/ipfs/QmdGSfmN4wWNXVs2XiwHbpjnUikJ7HyrTJNHyYGdodyJDC/go-ipfs-posinfo"
blockstore "gx/ipfs/QmdpuJBPBZ6sLPj9BQpn3Rpi38BT2cF1QMiUfyzNWeySW4/go-ipfs-blockstore"
ds "gx/ipfs/QmeiCcJfDW1GJnWUArudsv5rQsihpi4oyddPhdqo3CfX6i/go-datastore"
dsns "gx/ipfs/QmeiCcJfDW1GJnWUArudsv5rQsihpi4oyddPhdqo3CfX6i/go-datastore/namespace"
@ -121,7 +121,7 @@ func (f *FileManager) Get(c *cid.Cid) (blocks.Block, error) {
}
func (f *FileManager) readDataObj(c *cid.Cid, d *pb.DataObj) ([]byte, error) {
if !d.GetURL() {
if !IsURL(d.GetFilePath()) {
return f.readFileDataObj(c, d)
} else {
return f.readURLDataObj(c, d)
@ -256,7 +256,7 @@ func (f *FileManager) Put(b *posinfo.FilestoreNode) error {
func (f *FileManager) putTo(b *posinfo.FilestoreNode, to putter) error {
var dobj pb.DataObj
if !b.PosInfo.IsURL {
if !IsURL(b.PosInfo.FullPath) {
if !filepath.HasPrefix(b.PosInfo.FullPath, f.root) {
return fmt.Errorf("cannot add filestore references outside ipfs root (%s)", f.root)
}
@ -269,7 +269,6 @@ func (f *FileManager) putTo(b *posinfo.FilestoreNode, to putter) error {
dobj.FilePath = proto.String(filepath.ToSlash(p))
} else {
dobj.FilePath = proto.String(b.PosInfo.FullPath)
dobj.URL = proto.Bool(true)
}
dobj.Offset = proto.Uint64(b.PosInfo.Offset)
dobj.Size_ = proto.Uint64(uint64(len(b.RawData())))
@ -298,3 +297,9 @@ func (f *FileManager) PutMany(bs []*posinfo.FilestoreNode) error {
return batch.Commit()
}
func IsURL(str string) bool {
return (len(str) > 7 && str[0] == 'h' && str[1] == 't' && str[2] == 't' && str[3] == 'p') &&
((len(str) > 8 && str[4] == 's' && str[5] == ':' && str[6] == '/' && str[7] == '/') ||
(str[4] == ':' && str[5] == '/' && str[6] == '/'))
}

12
filestore/pb/Makefile
View File

@ -1,8 +1,10 @@
include mk/header.mk
PB = $(wildcard *.proto)
GO = $(PB:.proto=.pb.go)
PB_$(d) = $(wildcard $(d)/*.proto)
TGTS_$(d) = $(PB_$(d):.proto=.pb.go)
all: $(GO)
#DEPS_GO += $(TGTS_$(d))
%.pb.go: %.proto
protoc --gogo_out=. $<
include mk/footer.mk
clean:
rm *.pb.go

12
filestore/pb/dataobj.pb.go
View File

@ -1,12 +1,12 @@
// Code generated by protoc-gen-gogo.
// source: filestore/pb/dataobj.proto
// source: dataobj.proto
// DO NOT EDIT!
/*
Package datastore_pb is a generated protocol buffer package.
It is generated from these files:
filestore/pb/dataobj.proto
dataobj.proto
It has these top-level messages:
DataObj
@ -26,7 +26,6 @@ type DataObj struct {
FilePath *string `protobuf:"bytes,1,opt,name=FilePath" json:"FilePath,omitempty"`
Offset *uint64 `protobuf:"varint,2,opt,name=Offset" json:"Offset,omitempty"`
Size_ *uint64 `protobuf:"varint,3,opt,name=Size" json:"Size,omitempty"`
URL *bool `protobuf:"varint,4,opt,name=URL" json:"URL,omitempty"`
XXX_unrecognized []byte `json:"-"`
}
@ -55,13 +54,6 @@ func (m *DataObj) GetSize_() uint64 {
return 0
}
func (m *DataObj) GetURL() bool {
if m != nil && m.URL != nil {
return *m.URL
}
return false
}
func init() {
proto.RegisterType((*DataObj)(nil), "datastore.pb.DataObj")
}

1
filestore/pb/dataobj.proto
View File

@ -4,5 +4,4 @@ message DataObj {
optional string FilePath = 1;
optional uint64 Offset = 2;
optional uint64 Size = 3;
optional bool URL = 4;
}

4
importer/helpers/dagbuilder.go
View File

@ -25,7 +25,6 @@ type DagBuilderHelper struct {
maxlinks int
batch *ipld.Batch
fullPath string
isUrl bool
stat os.FileInfo
prefix *cid.Prefix
}
@ -71,7 +70,6 @@ func (dbp *DagBuilderParams) New(spl chunker.Splitter) *DagBuilderHelper {
if dbp.URL != "" {
db.fullPath = dbp.URL
db.isUrl = true
}
return db
}
@ -214,7 +212,7 @@ func (db *DagBuilderHelper) GetNextDataNode() (*UnixfsNode, error) {
// from the DagBuilderHelper.
func (db *DagBuilderHelper) SetPosInfo(node *UnixfsNode, offset uint64) {
if db.fullPath != "" {
node.SetPosInfo(offset, db.fullPath, db.stat, db.isUrl)
node.SetPosInfo(offset, db.fullPath, db.stat)
}
}

5
importer/helpers/helpers.go
View File

@ -8,9 +8,9 @@ import (
dag "github.com/ipfs/go-ipfs/merkledag"
ft "github.com/ipfs/go-ipfs/unixfs"
pi "gx/ipfs/QmUWsXLvYYDAaoAt9TPZpFX4ffHHMg46AHrz1ZLTN5ABbe/go-ipfs-posinfo"
ipld "gx/ipfs/QmWi2BYBL5gJ3CiAiQchg6rn1A8iBsrWy51EYxvHVjFvLb/go-ipld-format"
cid "gx/ipfs/QmapdYm1b22Frv3k17fqrBYTFRxwiaVJkB299Mfn33edeB/go-cid"
pi "gx/ipfs/QmdGSfmN4wWNXVs2XiwHbpjnUikJ7HyrTJNHyYGdodyJDC/go-ipfs-posinfo"
)
// BlockSizeLimit specifies the maximum size an imported block can have.
@ -142,12 +142,11 @@ func (n *UnixfsNode) FileSize() uint64 {
// SetPosInfo sets information about the offset of the data of this node in a
// filesystem file.
func (n *UnixfsNode) SetPosInfo(offset uint64, fullPath string, stat os.FileInfo, isUrl bool) {
func (n *UnixfsNode) SetPosInfo(offset uint64, fullPath string, stat os.FileInfo) {
n.posInfo = &pi.PosInfo{
Offset: offset,
FullPath: fullPath,
Stat: stat,
IsURL: isUrl,
}
}

4
package.json
View File

@ -422,9 +422,9 @@
},
{
"author": "hector",
"hash": "QmdGSfmN4wWNXVs2XiwHbpjnUikJ7HyrTJNHyYGdodyJDC",
"hash": "QmUWsXLvYYDAaoAt9TPZpFX4ffHHMg46AHrz1ZLTN5ABbe",
"name": "go-ipfs-posinfo",
"version": "0.0.3"
"version": "0.0.2"
},
{
"author": "hsanjuan",