From 989e91b1c457e710adc05b5eedde4a3bc28bd83b Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Fri, 24 Jun 2016 21:31:58 +0200 Subject: [PATCH 1/5] blockstore: add fetch rehashing License: MIT Signed-off-by: Jakub Sztandera --- blocks/blockstore/blockstore.go | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/blocks/blockstore/blockstore.go b/blocks/blockstore/blockstore.go index 3cc87a270..a1f1f600b 100644 --- a/blocks/blockstore/blockstore.go +++ b/blocks/blockstore/blockstore.go @@ -22,7 +22,8 @@ var log = logging.Logger("blockstore") // BlockPrefix namespaces blockstore datastores var BlockPrefix = ds.NewKey("blocks") -var ValueTypeMismatch = errors.New("The retrieved value is not a Block") +var ValueTypeMismatch = errors.New("the retrieved value is not a Block") +var ErrHashMismatch = errors.New("block in storage has different hash than requested") var ErrNotFound = errors.New("blockstore: block not found") @@ -71,6 +72,12 @@ type blockstore struct { lk sync.RWMutex gcreq int32 gcreqlk sync.Mutex + + rehash bool +} + +func (bs *blockstore) RuntimeHashing(enabled bool) { + bs.rehash = enabled } func (bs *blockstore) Get(k key.Key) (blocks.Block, error) { @@ -90,7 +97,16 @@ func (bs *blockstore) Get(k key.Key) (blocks.Block, error) { return nil, ValueTypeMismatch } - return blocks.NewBlockWithHash(bdata, mh.Multihash(k)) + if bs.rehash { + rb := blocks.NewBlock(bdata) + if rb.Key() != k { + return nil, ErrHashMismatch + } else { + return rb, nil + } + } else { + return blocks.NewBlockWithHash(bdata, mh.Multihash(k)) + } } func (bs *blockstore) Put(block blocks.Block) error { From f2686965ac5e93f3c672308a0fa467a36f0e2b1a Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Fri, 24 Jun 2016 22:03:16 +0200 Subject: [PATCH 2/5] tests: Add test to RuntimeHashing option of blockstore License: MIT Signed-off-by: Jakub Sztandera --- blocks/blockstore/blockstore_test.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/blocks/blockstore/blockstore_test.go b/blocks/blockstore/blockstore_test.go index 4a7eb7c74..babd1a99a 100644 --- a/blocks/blockstore/blockstore_test.go +++ b/blocks/blockstore/blockstore_test.go @@ -53,6 +53,22 @@ func TestPutThenGetBlock(t *testing.T) { } } +func TestRuntimeHashing(t *testing.T) { + bs := NewBlockstore(ds_sync.MutexWrap(ds.NewMapDatastore())) + bl := blocks.NewBlock([]byte("some data")) + blBad, err := blocks.NewBlockWithHash([]byte("some other data"), bl.Key().ToMultihash()) + if err != nil { + t.Fatal("Debug is enabled") + } + + bs.Put(blBad) + bs.RuntimeHashing(true) + + if _, err := bs.Get(bl.Key()); err != ErrHashMismatch { + t.Fatalf("Expected '%v' got '%v'\n", ErrHashMismatch, err) + } +} + func newBlockStoreWithKeys(t *testing.T, d ds.Datastore, N int) (Blockstore, []key.Key) { if d == nil { d = ds.NewMapDatastore() From 0f3dc1e84a717210c5de43201a0c64ef099cc49d Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Fri, 24 Jun 2016 22:10:31 +0200 Subject: [PATCH 3/5] core: Add config option for datastore read rehashing License: MIT Signed-off-by: Jakub Sztandera --- core/builder.go | 16 +++++++++++----- repo/config/datastore.go | 5 +++-- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/core/builder.go b/core/builder.go index c17d6ff6e..0d21f5b20 100644 --- a/core/builder.go +++ b/core/builder.go @@ -130,16 +130,22 @@ func setupNode(ctx context.Context, n *IpfsNode, cfg *BuildCfg) error { } var err error - n.Blockstore, err = bstore.WriteCached(bstore.NewBlockstore(n.Repo.Datastore()), kSizeBlockstoreWriteCache) + bs := bstore.NewBlockstore(n.Repo.Datastore()) + n.Blockstore, err = bstore.WriteCached(bs, kSizeBlockstoreWriteCache) if err != nil { return err } + rcfg, err := n.Repo.Config() + if err != nil { + return err + } + + if rcfg.Datastore.HashOnRead { + bs.RuntimeHashing(true) + } + if cfg.Online { - rcfg, err := n.Repo.Config() - if err != nil { - return err - } do := setupDiscoveryOption(rcfg.Discovery) if err := n.startOnlineServices(ctx, cfg.Routing, cfg.Host, do); err != nil { return err diff --git a/repo/config/datastore.go b/repo/config/datastore.go index 52582bd5c..2b9bf600d 100644 --- a/repo/config/datastore.go +++ b/repo/config/datastore.go @@ -15,8 +15,9 @@ type Datastore struct { StorageGCWatermark int64 // in percentage to multiply on StorageMax GCPeriod string // in ns, us, ms, s, m, h - Params *json.RawMessage - NoSync bool + Params *json.RawMessage + NoSync bool + HashOnRead bool } func (d *Datastore) ParamData() []byte { From 174ab12c67873678a0a5d8dd9c030f2e01dd0218 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Fri, 24 Jun 2016 22:22:38 +0200 Subject: [PATCH 4/5] config: Add explicit default value for HashOnRead License: MIT Signed-off-by: Jakub Sztandera --- repo/config/init.go | 1 + 1 file changed, 1 insertion(+) diff --git a/repo/config/init.go b/repo/config/init.go index ae5c813b7..8a6c72222 100644 --- a/repo/config/init.go +++ b/repo/config/init.go @@ -84,6 +84,7 @@ func datastoreConfig() (Datastore, error) { StorageMax: "10GB", StorageGCWatermark: 90, // 90% GCPeriod: "1h", + HashOnRead: false, }, nil } From dead77727227673ad00a9f27736201e82f666bbe Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Sun, 26 Jun 2016 21:32:59 +0200 Subject: [PATCH 5/5] Add sharness test for read blockstore rehashing License: MIT Signed-off-by: Jakub Sztandera --- test/sharness/t0084-repo-read-rehash.sh | 37 +++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100755 test/sharness/t0084-repo-read-rehash.sh diff --git a/test/sharness/t0084-repo-read-rehash.sh b/test/sharness/t0084-repo-read-rehash.sh new file mode 100755 index 000000000..f5bb5cb88 --- /dev/null +++ b/test/sharness/t0084-repo-read-rehash.sh @@ -0,0 +1,37 @@ +#!/bin/sh +# +# Copyright (c) Jakub Sztandera +# MIT Licensed; see the LICENSE file in this repository. +# + +test_description="Test ipfs blockstore repo read check." + +. lib/test-lib.sh + +rm -rf "$IPF_PATH/*" + +test_init_ipfs + + +H_BLOCK1=$(echo "Block 1" | ipfs add -q) +H_BLOCK2=$(echo "Block 2" | ipfs add -q) + +BS_BLOCK1="1220f18e/1220f18e07ebc69997909358f28b9d2c327eb032b0afab6bbc7fd7f399a7b7590be4.data" +BS_BLOCK2="1220dc58/1220dc582e51f1f98b1f2d1c1baaa9f7b11602239ed42fbdf8f52d67e63cc03df12a.data" + + +test_expect_success 'blocks are swapped' ' + ipfs cat $H_BLOCK2 > noswap && + cp -f "$IPFS_PATH/blocks/$BS_BLOCK1" "$IPFS_PATH/blocks/$BS_BLOCK2" && + ipfs cat $H_BLOCK2 > swap && + test_must_fail test_cmp noswap swap +' + +ipfs config --bool Datastore.HashOnRead true + +test_expect_success 'getting modified block fails' ' + (test_must_fail ipfs cat $H_BLOCK2 2> err_msg) && + grep "block in storage has different hash than requested" err_msg +' + +test_done