From 24daeec70c7c600e3f1c413fe6bf11cb57d5b243 Mon Sep 17 00:00:00 2001 From: Tommi Virtanen Date: Mon, 16 Mar 2015 14:03:28 -0700 Subject: [PATCH] Use flatfs to store objects under /blocks outside of LevelDB WARNING: No migration performed! That needs to come in a separate commit, perhaps amended into this one. Migration must move keyspace "/b" from leveldb to the flatfs subdir, while removing the "b" prefix (keys should start with just "/"). --- .../jbenet/go-datastore/flatfs/flatfs.go | 4 +-- .../jbenet/go-datastore/mount/mount.go | 6 ++-- blocks/blockstore/blockstore.go | 4 ++- repo/fsrepo/fsrepo.go | 35 ++++++++++++++++++- util/datastore2/threadsafe.go | 15 ++++++++ 5 files changed, 57 insertions(+), 7 deletions(-) create mode 100644 util/datastore2/threadsafe.go diff --git a/Godeps/_workspace/src/github.com/jbenet/go-datastore/flatfs/flatfs.go b/Godeps/_workspace/src/github.com/jbenet/go-datastore/flatfs/flatfs.go index c3bc96d4c..4704c3e1f 100644 --- a/Godeps/_workspace/src/github.com/jbenet/go-datastore/flatfs/flatfs.go +++ b/Godeps/_workspace/src/github.com/jbenet/go-datastore/flatfs/flatfs.go @@ -11,8 +11,8 @@ import ( "path" "strings" - "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore" - "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/query" + "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore" + "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/query" ) const ( diff --git a/Godeps/_workspace/src/github.com/jbenet/go-datastore/mount/mount.go b/Godeps/_workspace/src/github.com/jbenet/go-datastore/mount/mount.go index c066364a2..3c66cc3cf 100644 --- a/Godeps/_workspace/src/github.com/jbenet/go-datastore/mount/mount.go +++ b/Godeps/_workspace/src/github.com/jbenet/go-datastore/mount/mount.go @@ -6,9 +6,9 @@ import ( "errors" "strings" - "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore" - "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/keytransform" - "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/query" + "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore" + "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/keytransform" + "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/query" ) var ( diff --git a/blocks/blockstore/blockstore.go b/blocks/blockstore/blockstore.go index 7f3d4d7c8..ccc7e8fc5 100644 --- a/blocks/blockstore/blockstore.go +++ b/blocks/blockstore/blockstore.go @@ -18,7 +18,7 @@ import ( var log = eventlog.Logger("blockstore") // BlockPrefix namespaces blockstore datastores -var BlockPrefix = ds.NewKey("b") +var BlockPrefix = ds.NewKey("blocks") var ValueTypeMismatch = errors.New("The retrieved value is not a Block") @@ -89,6 +89,8 @@ func (bs *blockstore) AllKeysChan(ctx context.Context) (<-chan u.Key, error) { // KeysOnly, because that would be _a lot_ of data. q := dsq.Query{KeysOnly: true} + // datastore/namespace does *NOT* fix up Query.Prefix + q.Prefix = BlockPrefix.String() res, err := bs.datastore.Query(q) if err != nil { return nil, err diff --git a/repo/fsrepo/fsrepo.go b/repo/fsrepo/fsrepo.go index a9e750d54..ce856651d 100644 --- a/repo/fsrepo/fsrepo.go +++ b/repo/fsrepo/fsrepo.go @@ -10,7 +10,9 @@ import ( "sync" ds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore" + "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/flatfs" levelds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/leveldb" + "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/mount" ldbopts "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/opt" repo "github.com/ipfs/go-ipfs/repo" "github.com/ipfs/go-ipfs/repo/common" @@ -26,6 +28,7 @@ import ( const ( leveldbDirectory = "datastore" + flatfsDirectory = "blocks" ) var ( @@ -196,6 +199,11 @@ func Init(repoPath string, conf *config.Config) error { return fmt.Errorf("datastore: %s", err) } + flatfsPath := path.Join(repoPath, flatfsDirectory) + if err := dir.Writable(flatfsPath); err != nil { + return fmt.Errorf("datastore: %s", err) + } + if err := dir.Writable(path.Join(repoPath, "logs")); err != nil { return err } @@ -246,7 +254,32 @@ func (r *FSRepo) openDatastore() error { if err != nil { return errors.New("unable to open leveldb datastore") } - r.ds = r.leveldbDS + + // 4TB of 256kB objects ~=17M objects, splitting that 256-way + // leads to ~66k objects per dir, splitting 256*256-way leads to + // only 256. + // + // The keys seen by the block store have predictable prefixes, + // including "/" from datastore.Key and 2 bytes from multihash. To + // reach a uniform 256-way split, we need approximately 4 bytes of + // prefix. + blocksDS, err := flatfs.New(path.Join(r.path, flatfsDirectory), 4) + if err != nil { + return errors.New("unable to open flatfs datastore") + } + + mountDS := mount.New([]mount.Mount{ + {Prefix: ds.NewKey("/blocks"), Datastore: blocksDS}, + {Prefix: ds.NewKey("/"), Datastore: r.leveldbDS}, + }) + // Make sure it's ok to claim the virtual datastore from mount as + // threadsafe. There's no clean way to make mount itself provide + // this information without copy-pasting the code into two + // variants. This is the same dilemma as the `[].byte` attempt at + // introducing const types to Go. + var _ ds.ThreadSafeDatastore = blocksDS + var _ ds.ThreadSafeDatastore = r.leveldbDS + r.ds = ds2.ClaimThreadSafe{mountDS} return nil } diff --git a/util/datastore2/threadsafe.go b/util/datastore2/threadsafe.go new file mode 100644 index 000000000..2995943ca --- /dev/null +++ b/util/datastore2/threadsafe.go @@ -0,0 +1,15 @@ +package datastore2 + +import ( + "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore" +) + +// ClaimThreadSafe claims that a Datastore is threadsafe, even when +// it's type does not guarantee this. Use carefully. +type ClaimThreadSafe struct { + datastore.Datastore +} + +var _ datastore.ThreadSafeDatastore = ClaimThreadSafe{} + +func (ClaimThreadSafe) IsThreadSafe() {}