From 696a0f039e464f4c8b8db6dfa78c4480d5e268fd Mon Sep 17 00:00:00 2001 From: Kevin Atkinson Date: Fri, 22 Jun 2018 21:39:15 -0400 Subject: [PATCH] Simplify code: use prefix instead of flag to determine if a url License: MIT Signed-off-by: Kevin Atkinson --- core/coreunix/add.go | 2 +- core/coreunix/add_test.go | 2 +- filestore/filestore.go | 2 +- filestore/filestore_test.go | 14 +++++++++++++- filestore/fsrefstore.go | 13 +++++++++---- filestore/pb/Makefile | 12 +++++++----- filestore/pb/dataobj.pb.go | 12 ++---------- filestore/pb/dataobj.proto | 1 - importer/helpers/dagbuilder.go | 4 +--- importer/helpers/helpers.go | 5 ++--- package.json | 4 ++-- 11 files changed, 39 insertions(+), 32 deletions(-) diff --git a/core/coreunix/add.go b/core/coreunix/add.go index 5cfbbb486..88cd9e4a7 100644 --- a/core/coreunix/add.go +++ b/core/coreunix/add.go @@ -19,12 +19,12 @@ import ( "github.com/ipfs/go-ipfs/pin" unixfs "github.com/ipfs/go-ipfs/unixfs" + posinfo "gx/ipfs/QmUWsXLvYYDAaoAt9TPZpFX4ffHHMg46AHrz1ZLTN5ABbe/go-ipfs-posinfo" ipld "gx/ipfs/QmWi2BYBL5gJ3CiAiQchg6rn1A8iBsrWy51EYxvHVjFvLb/go-ipld-format" chunker "gx/ipfs/QmXnzH7wowyLZy8XJxxaQCVTgLMcDXdMBznmsrmQWCyiQV/go-ipfs-chunker" cid "gx/ipfs/QmapdYm1b22Frv3k17fqrBYTFRxwiaVJkB299Mfn33edeB/go-cid" logging "gx/ipfs/QmcVVHfdyv15GVPk7NrxdWjh2hLVccXnoD8j2tyQShiXJb/go-log" files "gx/ipfs/QmdE4gMduCKCGAcczM2F5ioYDfdeKuPix138wrES1YSr7f/go-ipfs-cmdkit/files" - posinfo "gx/ipfs/QmdGSfmN4wWNXVs2XiwHbpjnUikJ7HyrTJNHyYGdodyJDC/go-ipfs-posinfo" bstore "gx/ipfs/QmdpuJBPBZ6sLPj9BQpn3Rpi38BT2cF1QMiUfyzNWeySW4/go-ipfs-blockstore" ) diff --git a/core/coreunix/add_test.go b/core/coreunix/add_test.go index 1f2debc21..791289e2f 100644 --- a/core/coreunix/add_test.go +++ b/core/coreunix/add_test.go @@ -18,9 +18,9 @@ import ( "github.com/ipfs/go-ipfs/repo/config" blocks "gx/ipfs/QmTRCUvZLiir12Qr6MV3HKfKMHX8Nf1Vddn6t2g5nsQSb9/go-block-format" + pi "gx/ipfs/QmUWsXLvYYDAaoAt9TPZpFX4ffHHMg46AHrz1ZLTN5ABbe/go-ipfs-posinfo" cid "gx/ipfs/QmapdYm1b22Frv3k17fqrBYTFRxwiaVJkB299Mfn33edeB/go-cid" files "gx/ipfs/QmdE4gMduCKCGAcczM2F5ioYDfdeKuPix138wrES1YSr7f/go-ipfs-cmdkit/files" - pi "gx/ipfs/QmdGSfmN4wWNXVs2XiwHbpjnUikJ7HyrTJNHyYGdodyJDC/go-ipfs-posinfo" blockstore "gx/ipfs/QmdpuJBPBZ6sLPj9BQpn3Rpi38BT2cF1QMiUfyzNWeySW4/go-ipfs-blockstore" datastore "gx/ipfs/QmeiCcJfDW1GJnWUArudsv5rQsihpi4oyddPhdqo3CfX6i/go-datastore" syncds "gx/ipfs/QmeiCcJfDW1GJnWUArudsv5rQsihpi4oyddPhdqo3CfX6i/go-datastore/sync" diff --git a/filestore/filestore.go b/filestore/filestore.go index 6140451ed..66289b068 100644 --- a/filestore/filestore.go +++ b/filestore/filestore.go @@ -11,9 +11,9 @@ import ( "context" blocks "gx/ipfs/QmTRCUvZLiir12Qr6MV3HKfKMHX8Nf1Vddn6t2g5nsQSb9/go-block-format" + posinfo "gx/ipfs/QmUWsXLvYYDAaoAt9TPZpFX4ffHHMg46AHrz1ZLTN5ABbe/go-ipfs-posinfo" cid "gx/ipfs/QmapdYm1b22Frv3k17fqrBYTFRxwiaVJkB299Mfn33edeB/go-cid" logging "gx/ipfs/QmcVVHfdyv15GVPk7NrxdWjh2hLVccXnoD8j2tyQShiXJb/go-log" - posinfo "gx/ipfs/QmdGSfmN4wWNXVs2XiwHbpjnUikJ7HyrTJNHyYGdodyJDC/go-ipfs-posinfo" blockstore "gx/ipfs/QmdpuJBPBZ6sLPj9BQpn3Rpi38BT2cF1QMiUfyzNWeySW4/go-ipfs-blockstore" dsq "gx/ipfs/QmeiCcJfDW1GJnWUArudsv5rQsihpi4oyddPhdqo3CfX6i/go-datastore/query" ) diff --git a/filestore/filestore_test.go b/filestore/filestore_test.go index 6323f49c8..8336010e5 100644 --- a/filestore/filestore_test.go +++ b/filestore/filestore_test.go @@ -9,8 +9,8 @@ import ( dag "github.com/ipfs/go-ipfs/merkledag" + posinfo "gx/ipfs/QmUWsXLvYYDAaoAt9TPZpFX4ffHHMg46AHrz1ZLTN5ABbe/go-ipfs-posinfo" cid "gx/ipfs/QmapdYm1b22Frv3k17fqrBYTFRxwiaVJkB299Mfn33edeB/go-cid" - posinfo "gx/ipfs/QmdGSfmN4wWNXVs2XiwHbpjnUikJ7HyrTJNHyYGdodyJDC/go-ipfs-posinfo" blockstore "gx/ipfs/QmdpuJBPBZ6sLPj9BQpn3Rpi38BT2cF1QMiUfyzNWeySW4/go-ipfs-blockstore" ds "gx/ipfs/QmeiCcJfDW1GJnWUArudsv5rQsihpi4oyddPhdqo3CfX6i/go-datastore" ) @@ -162,3 +162,15 @@ func TestDeletes(t *testing.T) { } } } + +func TestIsURL(t *testing.T) { + if !IsURL("http://www.example.com") { + t.Fatal("IsURL failed: http://www.example.com") + } + if !IsURL("https://www.example.com") { + t.Fatal("IsURL failed: https://www.example.com") + } + if IsURL("adir/afile") { + t.Fatal("IsURL recognized non-url") + } +} diff --git a/filestore/fsrefstore.go b/filestore/fsrefstore.go index 6d4509876..d51139536 100644 --- a/filestore/fsrefstore.go +++ b/filestore/fsrefstore.go @@ -13,8 +13,8 @@ import ( dshelp "gx/ipfs/QmNP2u7bofwUQptHQGPfabGWtTCbxhNLSZKqbf1uzsup9V/go-ipfs-ds-help" proto "gx/ipfs/QmT6n4mspWYEya864BhCUJEgyxiRfmiSY9ruQwTUNpRKaM/protobuf/proto" blocks "gx/ipfs/QmTRCUvZLiir12Qr6MV3HKfKMHX8Nf1Vddn6t2g5nsQSb9/go-block-format" + posinfo "gx/ipfs/QmUWsXLvYYDAaoAt9TPZpFX4ffHHMg46AHrz1ZLTN5ABbe/go-ipfs-posinfo" cid "gx/ipfs/QmapdYm1b22Frv3k17fqrBYTFRxwiaVJkB299Mfn33edeB/go-cid" - posinfo "gx/ipfs/QmdGSfmN4wWNXVs2XiwHbpjnUikJ7HyrTJNHyYGdodyJDC/go-ipfs-posinfo" blockstore "gx/ipfs/QmdpuJBPBZ6sLPj9BQpn3Rpi38BT2cF1QMiUfyzNWeySW4/go-ipfs-blockstore" ds "gx/ipfs/QmeiCcJfDW1GJnWUArudsv5rQsihpi4oyddPhdqo3CfX6i/go-datastore" dsns "gx/ipfs/QmeiCcJfDW1GJnWUArudsv5rQsihpi4oyddPhdqo3CfX6i/go-datastore/namespace" @@ -121,7 +121,7 @@ func (f *FileManager) Get(c *cid.Cid) (blocks.Block, error) { } func (f *FileManager) readDataObj(c *cid.Cid, d *pb.DataObj) ([]byte, error) { - if !d.GetURL() { + if !IsURL(d.GetFilePath()) { return f.readFileDataObj(c, d) } else { return f.readURLDataObj(c, d) @@ -256,7 +256,7 @@ func (f *FileManager) Put(b *posinfo.FilestoreNode) error { func (f *FileManager) putTo(b *posinfo.FilestoreNode, to putter) error { var dobj pb.DataObj - if !b.PosInfo.IsURL { + if !IsURL(b.PosInfo.FullPath) { if !filepath.HasPrefix(b.PosInfo.FullPath, f.root) { return fmt.Errorf("cannot add filestore references outside ipfs root (%s)", f.root) } @@ -269,7 +269,6 @@ func (f *FileManager) putTo(b *posinfo.FilestoreNode, to putter) error { dobj.FilePath = proto.String(filepath.ToSlash(p)) } else { dobj.FilePath = proto.String(b.PosInfo.FullPath) - dobj.URL = proto.Bool(true) } dobj.Offset = proto.Uint64(b.PosInfo.Offset) dobj.Size_ = proto.Uint64(uint64(len(b.RawData()))) @@ -298,3 +297,9 @@ func (f *FileManager) PutMany(bs []*posinfo.FilestoreNode) error { return batch.Commit() } + +func IsURL(str string) bool { + return (len(str) > 7 && str[0] == 'h' && str[1] == 't' && str[2] == 't' && str[3] == 'p') && + ((len(str) > 8 && str[4] == 's' && str[5] == ':' && str[6] == '/' && str[7] == '/') || + (str[4] == ':' && str[5] == '/' && str[6] == '/')) +} diff --git a/filestore/pb/Makefile b/filestore/pb/Makefile index 505f70e75..5101a482d 100644 --- a/filestore/pb/Makefile +++ b/filestore/pb/Makefile @@ -1,8 +1,10 @@ -include mk/header.mk +PB = $(wildcard *.proto) +GO = $(PB:.proto=.pb.go) -PB_$(d) = $(wildcard $(d)/*.proto) -TGTS_$(d) = $(PB_$(d):.proto=.pb.go) +all: $(GO) -#DEPS_GO += $(TGTS_$(d)) +%.pb.go: %.proto + protoc --gogo_out=. $< -include mk/footer.mk +clean: + rm *.pb.go diff --git a/filestore/pb/dataobj.pb.go b/filestore/pb/dataobj.pb.go index 517ae1b7f..fadd40c1a 100644 --- a/filestore/pb/dataobj.pb.go +++ b/filestore/pb/dataobj.pb.go @@ -1,12 +1,12 @@ // Code generated by protoc-gen-gogo. -// source: filestore/pb/dataobj.proto +// source: dataobj.proto // DO NOT EDIT! /* Package datastore_pb is a generated protocol buffer package. It is generated from these files: - filestore/pb/dataobj.proto + dataobj.proto It has these top-level messages: DataObj @@ -26,7 +26,6 @@ type DataObj struct { FilePath *string `protobuf:"bytes,1,opt,name=FilePath" json:"FilePath,omitempty"` Offset *uint64 `protobuf:"varint,2,opt,name=Offset" json:"Offset,omitempty"` Size_ *uint64 `protobuf:"varint,3,opt,name=Size" json:"Size,omitempty"` - URL *bool `protobuf:"varint,4,opt,name=URL" json:"URL,omitempty"` XXX_unrecognized []byte `json:"-"` } @@ -55,13 +54,6 @@ func (m *DataObj) GetSize_() uint64 { return 0 } -func (m *DataObj) GetURL() bool { - if m != nil && m.URL != nil { - return *m.URL - } - return false -} - func init() { proto.RegisterType((*DataObj)(nil), "datastore.pb.DataObj") } diff --git a/filestore/pb/dataobj.proto b/filestore/pb/dataobj.proto index 311042a0e..c7d7f0eea 100644 --- a/filestore/pb/dataobj.proto +++ b/filestore/pb/dataobj.proto @@ -4,5 +4,4 @@ message DataObj { optional string FilePath = 1; optional uint64 Offset = 2; optional uint64 Size = 3; - optional bool URL = 4; } diff --git a/importer/helpers/dagbuilder.go b/importer/helpers/dagbuilder.go index d58129ab4..5dac1b569 100644 --- a/importer/helpers/dagbuilder.go +++ b/importer/helpers/dagbuilder.go @@ -25,7 +25,6 @@ type DagBuilderHelper struct { maxlinks int batch *ipld.Batch fullPath string - isUrl bool stat os.FileInfo prefix *cid.Prefix } @@ -71,7 +70,6 @@ func (dbp *DagBuilderParams) New(spl chunker.Splitter) *DagBuilderHelper { if dbp.URL != "" { db.fullPath = dbp.URL - db.isUrl = true } return db } @@ -214,7 +212,7 @@ func (db *DagBuilderHelper) GetNextDataNode() (*UnixfsNode, error) { // from the DagBuilderHelper. func (db *DagBuilderHelper) SetPosInfo(node *UnixfsNode, offset uint64) { if db.fullPath != "" { - node.SetPosInfo(offset, db.fullPath, db.stat, db.isUrl) + node.SetPosInfo(offset, db.fullPath, db.stat) } } diff --git a/importer/helpers/helpers.go b/importer/helpers/helpers.go index 9fae2c745..64a22f551 100644 --- a/importer/helpers/helpers.go +++ b/importer/helpers/helpers.go @@ -8,9 +8,9 @@ import ( dag "github.com/ipfs/go-ipfs/merkledag" ft "github.com/ipfs/go-ipfs/unixfs" + pi "gx/ipfs/QmUWsXLvYYDAaoAt9TPZpFX4ffHHMg46AHrz1ZLTN5ABbe/go-ipfs-posinfo" ipld "gx/ipfs/QmWi2BYBL5gJ3CiAiQchg6rn1A8iBsrWy51EYxvHVjFvLb/go-ipld-format" cid "gx/ipfs/QmapdYm1b22Frv3k17fqrBYTFRxwiaVJkB299Mfn33edeB/go-cid" - pi "gx/ipfs/QmdGSfmN4wWNXVs2XiwHbpjnUikJ7HyrTJNHyYGdodyJDC/go-ipfs-posinfo" ) // BlockSizeLimit specifies the maximum size an imported block can have. @@ -142,12 +142,11 @@ func (n *UnixfsNode) FileSize() uint64 { // SetPosInfo sets information about the offset of the data of this node in a // filesystem file. -func (n *UnixfsNode) SetPosInfo(offset uint64, fullPath string, stat os.FileInfo, isUrl bool) { +func (n *UnixfsNode) SetPosInfo(offset uint64, fullPath string, stat os.FileInfo) { n.posInfo = &pi.PosInfo{ Offset: offset, FullPath: fullPath, Stat: stat, - IsURL: isUrl, } } diff --git a/package.json b/package.json index 0395f5dae..748f889b2 100644 --- a/package.json +++ b/package.json @@ -422,9 +422,9 @@ }, { "author": "hector", - "hash": "QmdGSfmN4wWNXVs2XiwHbpjnUikJ7HyrTJNHyYGdodyJDC", + "hash": "QmUWsXLvYYDAaoAt9TPZpFX4ffHHMg46AHrz1ZLTN5ABbe", "name": "go-ipfs-posinfo", - "version": "0.0.3" + "version": "0.0.2" }, { "author": "hsanjuan",