opensource/kubo
1
0
Fork 0
mirror of https://github.com/ipfs/kubo.git synced 2025-06-30 09:59:13 +08:00
Code Issues Projects Releases Wiki Activity

Merge pull request #3830 from ipfs/golint-2-filestore

Browse Source 
Filestore: make golint happy
This commit is contained in:
Jeromy Johnson
2017-03-24 14:34:07 -07:00
committed by GitHub
parent b1d28f1b76 8c40bd9194
commit 1ea03545fe
3 changed files with 94 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
filestore/filestore.go
View File

@ -1,3 +1,10 @@
// Package filestore implements a Blockstore which is able to read certain
// blocks of data directly from its original location in the filesystem.
//
// In a Filestore, object leaves are stored as FilestoreNodes. FilestoreNodes
// include a filesystem path and an offset, allowing a Blockstore dealing with
// such blocks to avoid storing the whole contents and reading them from their
// filesystem location instead.
package filestore
import (
@ -14,23 +21,31 @@ import (
var log = logging.Logger("filestore")
// Filestore implements a Blockstore by combining a standard Blockstore
// to store regular blocks and a special Blockstore called
// FileManager to store blocks which data exists in an external file.
type Filestore struct {
fm *FileManager
bs blockstore.Blockstore
}
// FileManager returns the FileManager in Filestore.
func (f *Filestore) FileManager() *FileManager {
return f.fm
}
// MainBlockstore returns the standard Blockstore in the Filestore.
func (f *Filestore) MainBlockstore() blockstore.Blockstore {
return f.bs
}
// NewFilestore creates one using the given Blockstore and FileManager.
func NewFilestore(bs blockstore.Blockstore, fm *FileManager) *Filestore {
return &Filestore{fm, bs}
}
// AllKeysChan returns a channel from which to read the keys stored in
// the blockstore. If the given context is cancelled the channel will be closed.
func (f *Filestore) AllKeysChan(ctx context.Context) (<-chan *cid.Cid, error) {
ctx, cancel := context.WithCancel(ctx)
@ -93,6 +108,10 @@ func (f *Filestore) AllKeysChan(ctx context.Context) (<-chan *cid.Cid, error) {
return out, nil
}
// DeleteBlock deletes the block with the given key from the
// blockstore. As expected, in the case of FileManager blocks, only the
// reference is deleted, not its contents. It may return
// ErrNotFound when the block is not stored.
func (f *Filestore) DeleteBlock(c *cid.Cid) error {
err1 := f.bs.DeleteBlock(c)
if err1 != nil && err1 != blockstore.ErrNotFound {
@ -116,6 +135,8 @@ func (f *Filestore) DeleteBlock(c *cid.Cid) error {
}
}
// Get retrieves the block with the given Cid. It may return
// ErrNotFound when the block is not stored.
func (f *Filestore) Get(c *cid.Cid) (blocks.Block, error) {
blk, err := f.bs.Get(c)
switch err {
@ -130,6 +151,8 @@ func (f *Filestore) Get(c *cid.Cid) (blocks.Block, error) {
return f.fm.Get(c)
}
// Has returns true if the block with the given Cid is
// stored in the Filestore.
func (f *Filestore) Has(c *cid.Cid) (bool, error) {
has, err := f.bs.Has(c)
if err != nil {
@ -143,6 +166,10 @@ func (f *Filestore) Has(c *cid.Cid) (bool, error) {
return f.fm.Has(c)
}
// Put stores a block in the Filestore. For blocks of
// underlying type FilestoreNode, the operation is
// delegated to the FileManager, while the rest of blocks
// are handled by the regular blockstore.
func (f *Filestore) Put(b blocks.Block) error {
has, err := f.Has(b.Cid())
if err != nil {
@ -161,6 +188,8 @@ func (f *Filestore) Put(b blocks.Block) error {
}
}
// PutMany is like Put(), but takes a slice of blocks, allowing
// the underlying blockstore to perform batch transactions.
func (f *Filestore) PutMany(bs []blocks.Block) error {
var normals []blocks.Block
var fstores []*posinfo.FilestoreNode

29
filestore/fsrefstore.go
View File

@ -20,26 +20,43 @@ import (
cid "gx/ipfs/QmV5gPoRsjN1Gid3LMdNZTyfCtP2DsvqEbMAmz82RmmiGk/go-cid"
)
// FilestorePrefix identifies the key prefix for FileManager blocks.
var FilestorePrefix = ds.NewKey("filestore")
// FileManager is a blockstore implementation which stores special
// blocks FilestoreNode type. These nodes only contain a reference
// to the actual location of the block data in the filesystem
// (a path and an offset).
type FileManager struct {
ds ds.Batching
root string
}
// CorruptReferenceError implements the error interface.
// It is used to indicate that the block contents pointed
// by the referencing blocks cannot be retrieved (i.e. the
// file is not found, or the data changed as it was being read).
type CorruptReferenceError struct {
Code Status
Err error
}
// Error() returns the error message in the CorruptReferenceError
// as a string.
func (c CorruptReferenceError) Error() string {
return c.Err.Error()
}
// NewFileManager initializes a new file manager with the given
// datastore and root. All FilestoreNodes paths are relative to the
// root path given here, which is prepended for any operations.
func NewFileManager(ds ds.Batching, root string) *FileManager {
return &FileManager{dsns.Wrap(ds, FilestorePrefix), root}
}
// AllKeysChan returns a channel from which to read the keys stored in
// the FileManager. If the given context is cancelled the channel will be
// closed.
func (f *FileManager) AllKeysChan(ctx context.Context) (<-chan *cid.Cid, error) {
q := dsq.Query{KeysOnly: true}
q.Prefix = FilestorePrefix.String()
@ -76,6 +93,8 @@ func (f *FileManager) AllKeysChan(ctx context.Context) (<-chan *cid.Cid, error)
return out, nil
}
// DeleteBlock deletes the reference-block from the underlying
// datastore. It does not touch the referenced data.
func (f *FileManager) DeleteBlock(c *cid.Cid) error {
err := f.ds.Delete(dshelp.CidToDsKey(c))
if err == ds.ErrNotFound {
@ -84,6 +103,10 @@ func (f *FileManager) DeleteBlock(c *cid.Cid) error {
return err
}
// Get reads a block from the datastore. Reading a block
// is done in two steps: the first step retrieves the reference
// block from the datastore. The second step uses the stored
// path and offsets to read the raw block data directly from disk.
func (f *FileManager) Get(c *cid.Cid) (blocks.Block, error) {
dobj, err := f.getDataObj(c)
if err != nil {
@ -165,6 +188,8 @@ func (f *FileManager) readDataObj(c *cid.Cid, d *pb.DataObj) ([]byte, error) {
return outbuf, nil
}
// Has returns if the FileManager is storing a block reference. It does not
// validate the data, nor checks if the reference is valid.
func (f *FileManager) Has(c *cid.Cid) (bool, error) {
// NOTE: interesting thing to consider. Has doesnt validate the data.
// So the data on disk could be invalid, and we could think we have it.
@ -176,6 +201,8 @@ type putter interface {
Put(ds.Key, interface{}) error
}
// Put adds a new reference block to the FileManager. It does not check
// that the reference is valid.
func (f *FileManager) Put(b *posinfo.FilestoreNode) error {
return f.putTo(b, f.ds)
}
@ -204,6 +231,8 @@ func (f *FileManager) putTo(b *posinfo.FilestoreNode, to putter) error {
return to.Put(dshelp.CidToDsKey(b.Cid()), data)
}
// PutMany is like Put() but takes a slice of blocks instead,
// allowing it to create a batch transaction.
func (f *FileManager) PutMany(bs []*posinfo.FilestoreNode) error {
batch, err := f.ds.Batch()
if err != nil {

45
filestore/util.go
View File

@ -12,8 +12,11 @@ import (
cid "gx/ipfs/QmV5gPoRsjN1Gid3LMdNZTyfCtP2DsvqEbMAmz82RmmiGk/go-cid"
)
// Status is used to identify the state of the block data referenced
// by a FilestoreNode. Among other places, it is used by CorruptReferenceError.
type Status int32
// These are the supported Status codes.
const (
StatusOk Status = 0
StatusFileError Status = 10 // Backing File Error
@ -23,6 +26,7 @@ const (
StatusKeyNotFound Status = 30
)
// String provides a human-readable representation for Status codes.
func (s Status) String() string {
switch s {
case StatusOk:
@ -42,10 +46,16 @@ func (s Status) String() string {
}
}
// Format returns the status formatted as a string
// with leading 0s.
func (s Status) Format() string {
return fmt.Sprintf("%-7s", s.String())
}
// ListRes wraps the response of the List*() functions, which
// allows to obtain and verify blocks stored by the FileManager
// of a Filestore. It includes information about the referenced
// block.
type ListRes struct {
Status Status
ErrorMsg string
@ -55,6 +65,7 @@ type ListRes struct {
Size uint64
}
// FormatLong returns a human readable string for a ListRes object.
func (r *ListRes) FormatLong() string {
switch {
case r.Key == nil:
@ -66,18 +77,34 @@ func (r *ListRes) FormatLong() string {
}
}
// List fetches the block with the given key from the Filemanager
// of the given Filestore and returns a ListRes object with the information.
// List does not verify that the reference is valid or whether the
// raw data is accesible. See Verify().
func List(fs *Filestore, key *cid.Cid) *ListRes {
return list(fs, false, key)
}
// ListAll returns a function as an iterator which, once invoked, returns
// one by one each block in the Filestore's FileManager.
// ListAll does not verify that the references are valid or whether
// the raw data is accessible. See VerifyAll().
func ListAll(fs *Filestore) (func() *ListRes, error) {
return listAll(fs, false)
}
// Verify fetches the block with the given key from the Filemanager
// of the given Filestore and returns a ListRes object with the information.
// Verify makes sure that the reference is valid and the block data can be
// read.
func Verify(fs *Filestore, key *cid.Cid) *ListRes {
return list(fs, true, key)
}
// VerifyAll returns a function as an iterator which, once invoked,
// returns one by one each block in the Filestore's FileManager.
// VerifyAll checks that the reference is valid and that the block data
// can be read.
func VerifyAll(fs *Filestore) (func() *ListRes, error) {
return listAll(fs, true)
}
@ -150,14 +177,14 @@ func mkListRes(c *cid.Cid, d *pb.DataObj, err error) *ListRes {
ErrorMsg: errorMsg,
Key: c,
}
} else {
return &ListRes{
Status: status,
ErrorMsg: errorMsg,
Key: c,
FilePath: *d.FilePath,
Size: *d.Size_,
Offset: *d.Offset,
}
}
return &ListRes{
Status: status,
ErrorMsg: errorMsg,
Key: c,
FilePath: *d.FilePath,
Size: *d.Size_,
Offset: *d.Offset,
}
}