From 8c40bd91946399e53b5a8c3f78f3d8b96c1802b1 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Fri, 24 Mar 2017 20:36:00 +0100 Subject: [PATCH] Filestore: make golint happy Comments for exported functions and little else. License: MIT Signed-off-by: Hector Sanjuan --- filestore/filestore.go | 29 ++++++++++++++++++++++++++ filestore/fsrefstore.go | 29 ++++++++++++++++++++++++++ filestore/util.go | 45 ++++++++++++++++++++++++++++++++--------- 3 files changed, 94 insertions(+), 9 deletions(-) diff --git a/filestore/filestore.go b/filestore/filestore.go index 801f78d1c..8c5822705 100644 --- a/filestore/filestore.go +++ b/filestore/filestore.go @@ -1,3 +1,10 @@ +// Package filestore implements a Blockstore which is able to read certain +// blocks of data directly from its original location in the filesystem. +// +// In a Filestore, object leaves are stored as FilestoreNodes. FilestoreNodes +// include a filesystem path and an offset, allowing a Blockstore dealing with +// such blocks to avoid storing the whole contents and reading them from their +// filesystem location instead. package filestore import ( @@ -14,23 +21,31 @@ import ( var log = logging.Logger("filestore") +// Filestore implements a Blockstore by combining a standard Blockstore +// to store regular blocks and a special Blockstore called +// FileManager to store blocks which data exists in an external file. type Filestore struct { fm *FileManager bs blockstore.Blockstore } +// FileManager returns the FileManager in Filestore. func (f *Filestore) FileManager() *FileManager { return f.fm } +// MainBlockstore returns the standard Blockstore in the Filestore. func (f *Filestore) MainBlockstore() blockstore.Blockstore { return f.bs } +// NewFilestore creates one using the given Blockstore and FileManager. func NewFilestore(bs blockstore.Blockstore, fm *FileManager) *Filestore { return &Filestore{fm, bs} } +// AllKeysChan returns a channel from which to read the keys stored in +// the blockstore. If the given context is cancelled the channel will be closed. func (f *Filestore) AllKeysChan(ctx context.Context) (<-chan *cid.Cid, error) { ctx, cancel := context.WithCancel(ctx) @@ -93,6 +108,10 @@ func (f *Filestore) AllKeysChan(ctx context.Context) (<-chan *cid.Cid, error) { return out, nil } +// DeleteBlock deletes the block with the given key from the +// blockstore. As expected, in the case of FileManager blocks, only the +// reference is deleted, not its contents. It may return +// ErrNotFound when the block is not stored. func (f *Filestore) DeleteBlock(c *cid.Cid) error { err1 := f.bs.DeleteBlock(c) if err1 != nil && err1 != blockstore.ErrNotFound { @@ -116,6 +135,8 @@ func (f *Filestore) DeleteBlock(c *cid.Cid) error { } } +// Get retrieves the block with the given Cid. It may return +// ErrNotFound when the block is not stored. func (f *Filestore) Get(c *cid.Cid) (blocks.Block, error) { blk, err := f.bs.Get(c) switch err { @@ -130,6 +151,8 @@ func (f *Filestore) Get(c *cid.Cid) (blocks.Block, error) { return f.fm.Get(c) } +// Has returns true if the block with the given Cid is +// stored in the Filestore. func (f *Filestore) Has(c *cid.Cid) (bool, error) { has, err := f.bs.Has(c) if err != nil { @@ -143,6 +166,10 @@ func (f *Filestore) Has(c *cid.Cid) (bool, error) { return f.fm.Has(c) } +// Put stores a block in the Filestore. For blocks of +// underlying type FilestoreNode, the operation is +// delegated to the FileManager, while the rest of blocks +// are handled by the regular blockstore. func (f *Filestore) Put(b blocks.Block) error { has, err := f.Has(b.Cid()) if err != nil { @@ -161,6 +188,8 @@ func (f *Filestore) Put(b blocks.Block) error { } } +// PutMany is like Put(), but takes a slice of blocks, allowing +// the underlying blockstore to perform batch transactions. func (f *Filestore) PutMany(bs []blocks.Block) error { var normals []blocks.Block var fstores []*posinfo.FilestoreNode diff --git a/filestore/fsrefstore.go b/filestore/fsrefstore.go index 3e3750cb9..7b41555b9 100644 --- a/filestore/fsrefstore.go +++ b/filestore/fsrefstore.go @@ -20,26 +20,43 @@ import ( cid "gx/ipfs/QmV5gPoRsjN1Gid3LMdNZTyfCtP2DsvqEbMAmz82RmmiGk/go-cid" ) +// FilestorePrefix identifies the key prefix for FileManager blocks. var FilestorePrefix = ds.NewKey("filestore") +// FileManager is a blockstore implementation which stores special +// blocks FilestoreNode type. These nodes only contain a reference +// to the actual location of the block data in the filesystem +// (a path and an offset). type FileManager struct { ds ds.Batching root string } +// CorruptReferenceError implements the error interface. +// It is used to indicate that the block contents pointed +// by the referencing blocks cannot be retrieved (i.e. the +// file is not found, or the data changed as it was being read). type CorruptReferenceError struct { Code Status Err error } +// Error() returns the error message in the CorruptReferenceError +// as a string. func (c CorruptReferenceError) Error() string { return c.Err.Error() } +// NewFileManager initializes a new file manager with the given +// datastore and root. All FilestoreNodes paths are relative to the +// root path given here, which is prepended for any operations. func NewFileManager(ds ds.Batching, root string) *FileManager { return &FileManager{dsns.Wrap(ds, FilestorePrefix), root} } +// AllKeysChan returns a channel from which to read the keys stored in +// the FileManager. If the given context is cancelled the channel will be +// closed. func (f *FileManager) AllKeysChan(ctx context.Context) (<-chan *cid.Cid, error) { q := dsq.Query{KeysOnly: true} q.Prefix = FilestorePrefix.String() @@ -76,6 +93,8 @@ func (f *FileManager) AllKeysChan(ctx context.Context) (<-chan *cid.Cid, error) return out, nil } +// DeleteBlock deletes the reference-block from the underlying +// datastore. It does not touch the referenced data. func (f *FileManager) DeleteBlock(c *cid.Cid) error { err := f.ds.Delete(dshelp.CidToDsKey(c)) if err == ds.ErrNotFound { @@ -84,6 +103,10 @@ func (f *FileManager) DeleteBlock(c *cid.Cid) error { return err } +// Get reads a block from the datastore. Reading a block +// is done in two steps: the first step retrieves the reference +// block from the datastore. The second step uses the stored +// path and offsets to read the raw block data directly from disk. func (f *FileManager) Get(c *cid.Cid) (blocks.Block, error) { dobj, err := f.getDataObj(c) if err != nil { @@ -165,6 +188,8 @@ func (f *FileManager) readDataObj(c *cid.Cid, d *pb.DataObj) ([]byte, error) { return outbuf, nil } +// Has returns if the FileManager is storing a block reference. It does not +// validate the data, nor checks if the reference is valid. func (f *FileManager) Has(c *cid.Cid) (bool, error) { // NOTE: interesting thing to consider. Has doesnt validate the data. // So the data on disk could be invalid, and we could think we have it. @@ -176,6 +201,8 @@ type putter interface { Put(ds.Key, interface{}) error } +// Put adds a new reference block to the FileManager. It does not check +// that the reference is valid. func (f *FileManager) Put(b *posinfo.FilestoreNode) error { return f.putTo(b, f.ds) } @@ -204,6 +231,8 @@ func (f *FileManager) putTo(b *posinfo.FilestoreNode, to putter) error { return to.Put(dshelp.CidToDsKey(b.Cid()), data) } +// PutMany is like Put() but takes a slice of blocks instead, +// allowing it to create a batch transaction. func (f *FileManager) PutMany(bs []*posinfo.FilestoreNode) error { batch, err := f.ds.Batch() if err != nil { diff --git a/filestore/util.go b/filestore/util.go index f098d2e17..0d764cfb7 100644 --- a/filestore/util.go +++ b/filestore/util.go @@ -12,8 +12,11 @@ import ( cid "gx/ipfs/QmV5gPoRsjN1Gid3LMdNZTyfCtP2DsvqEbMAmz82RmmiGk/go-cid" ) +// Status is used to identify the state of the block data referenced +// by a FilestoreNode. Among other places, it is used by CorruptReferenceError. type Status int32 +// These are the supported Status codes. const ( StatusOk Status = 0 StatusFileError Status = 10 // Backing File Error @@ -23,6 +26,7 @@ const ( StatusKeyNotFound Status = 30 ) +// String provides a human-readable representation for Status codes. func (s Status) String() string { switch s { case StatusOk: @@ -42,10 +46,16 @@ func (s Status) String() string { } } +// Format returns the status formatted as a string +// with leading 0s. func (s Status) Format() string { return fmt.Sprintf("%-7s", s.String()) } +// ListRes wraps the response of the List*() functions, which +// allows to obtain and verify blocks stored by the FileManager +// of a Filestore. It includes information about the referenced +// block. type ListRes struct { Status Status ErrorMsg string @@ -55,6 +65,7 @@ type ListRes struct { Size uint64 } +// FormatLong returns a human readable string for a ListRes object. func (r *ListRes) FormatLong() string { switch { case r.Key == nil: @@ -66,18 +77,34 @@ func (r *ListRes) FormatLong() string { } } +// List fetches the block with the given key from the Filemanager +// of the given Filestore and returns a ListRes object with the information. +// List does not verify that the reference is valid or whether the +// raw data is accesible. See Verify(). func List(fs *Filestore, key *cid.Cid) *ListRes { return list(fs, false, key) } +// ListAll returns a function as an iterator which, once invoked, returns +// one by one each block in the Filestore's FileManager. +// ListAll does not verify that the references are valid or whether +// the raw data is accessible. See VerifyAll(). func ListAll(fs *Filestore) (func() *ListRes, error) { return listAll(fs, false) } +// Verify fetches the block with the given key from the Filemanager +// of the given Filestore and returns a ListRes object with the information. +// Verify makes sure that the reference is valid and the block data can be +// read. func Verify(fs *Filestore, key *cid.Cid) *ListRes { return list(fs, true, key) } +// VerifyAll returns a function as an iterator which, once invoked, +// returns one by one each block in the Filestore's FileManager. +// VerifyAll checks that the reference is valid and that the block data +// can be read. func VerifyAll(fs *Filestore) (func() *ListRes, error) { return listAll(fs, true) } @@ -150,14 +177,14 @@ func mkListRes(c *cid.Cid, d *pb.DataObj, err error) *ListRes { ErrorMsg: errorMsg, Key: c, } - } else { - return &ListRes{ - Status: status, - ErrorMsg: errorMsg, - Key: c, - FilePath: *d.FilePath, - Size: *d.Size_, - Offset: *d.Offset, - } + } + + return &ListRes{ + Status: status, + ErrorMsg: errorMsg, + Key: c, + FilePath: *d.FilePath, + Size: *d.Size_, + Offset: *d.Offset, } }