diff --git a/blocks/set/set.go b/blocks/set/set.go index a2b83d840..6b7d744cb 100644 --- a/blocks/set/set.go +++ b/blocks/set/set.go @@ -1,10 +1,6 @@ package set import ( - "errors" - - ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore" - "github.com/jbenet/go-ipfs/blocks/bloom" "github.com/jbenet/go-ipfs/util" ) @@ -28,19 +24,6 @@ func SimpleSetFromKeys(keys []util.Key) BlockSet { return sbs } -func SetFromDatastore(d ds.Datastore, k ds.Key) (BlockSet, error) { - ikeys, err := d.Get(k) - if err != nil { - return nil, err - } - - keys, ok := ikeys.([]util.Key) - if !ok { - return nil, errors.New("Incorrect type for keys from datastore") - } - return SimpleSetFromKeys(keys), nil -} - func NewSimpleBlockSet() BlockSet { return &simpleBlockSet{blocks: make(map[util.Key]struct{})} } @@ -64,7 +47,7 @@ func (b *simpleBlockSet) HasKey(k util.Key) bool { func (b *simpleBlockSet) GetBloomFilter() bloom.Filter { f := bloom.BasicFilter() - for k, _ := range b.blocks { + for k := range b.blocks { f.Add([]byte(k)) } return f @@ -72,7 +55,7 @@ func (b *simpleBlockSet) GetBloomFilter() bloom.Filter { func (b *simpleBlockSet) GetKeys() []util.Key { var out []util.Key - for k, _ := range b.blocks { + for k := range b.blocks { out = append(out, k) } return out diff --git a/pin/indirect.go b/pin/indirect.go index 27b11292c..2eb303de2 100644 --- a/pin/indirect.go +++ b/pin/indirect.go @@ -1,8 +1,6 @@ package pin import ( - "errors" - ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore" "github.com/jbenet/go-ipfs/blocks/set" "github.com/jbenet/go-ipfs/util" @@ -21,23 +19,33 @@ func NewIndirectPin(dstore ds.Datastore) *indirectPin { } func loadIndirPin(d ds.Datastore, k ds.Key) (*indirectPin, error) { - irefcnt, err := d.Get(k) + var rcStore map[string]int + err := loadSet(d, k, &rcStore) if err != nil { return nil, err } - refcnt, ok := irefcnt.(map[util.Key]int) - if !ok { - return nil, errors.New("invalid type from datastore") - } + refcnt := make(map[util.Key]int) var keys []util.Key - for k, _ := range refcnt { + for encK, v := range rcStore { + k := util.B58KeyDecode(encK) keys = append(keys, k) + refcnt[k] = v } + log.Debug("indirPin keys: %#v", keys) return &indirectPin{blockset: set.SimpleSetFromKeys(keys), refCounts: refcnt}, nil } +func storeIndirPin(d ds.Datastore, k ds.Key, p *indirectPin) error { + + rcStore := map[string]int{} + for k, v := range p.refCounts { + rcStore[util.B58KeyEncode(k)] = v + } + return storeSet(d, k, rcStore) +} + func (i *indirectPin) Increment(k util.Key) { c := i.refCounts[k] i.refCounts[k] = c + 1 diff --git a/pin/pin.go b/pin/pin.go index 1c02fd038..b9c509a03 100644 --- a/pin/pin.go +++ b/pin/pin.go @@ -3,8 +3,9 @@ package pin import ( //ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go" - "bytes" + "encoding/json" + "errors" "sync" ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore" @@ -14,6 +15,7 @@ import ( "github.com/jbenet/go-ipfs/util" ) +var log = util.Logger("pin") var recursePinDatastoreKey = ds.NewKey("/local/pins/recursive/keys") var directPinDatastoreKey = ds.NewKey("/local/pins/direct/keys") var indirectPinDatastoreKey = ds.NewKey("/local/pins/indirect/keys") @@ -89,9 +91,8 @@ func (p *pinner) Unpin(k util.Key, recurse bool) error { } return p.unpinLinks(node) - } else { - p.directPin.RemoveBlock(k) } + p.directPin.RemoveBlock(k) return nil } @@ -153,21 +154,31 @@ func (p *pinner) IsPinned(key util.Key) bool { func LoadPinner(d ds.Datastore, dserv *mdag.DAGService) (Pinner, error) { p := new(pinner) - var err error - p.recursePin, err = set.SetFromDatastore(d, recursePinDatastoreKey) - if err != nil { - return nil, err - } - p.directPin, err = set.SetFromDatastore(d, directPinDatastoreKey) - if err != nil { - return nil, err + { // load recursive set + var recurseKeys []util.Key + if err := loadSet(d, recursePinDatastoreKey, &recurseKeys); err != nil { + return nil, err + } + p.recursePin = set.SimpleSetFromKeys(recurseKeys) } - p.indirPin, err = loadIndirPin(d, indirectPinDatastoreKey) - if err != nil { - return nil, err + { // load direct set + var directKeys []util.Key + if err := loadSet(d, directPinDatastoreKey, &directKeys); err != nil { + return nil, err + } + p.directPin = set.SimpleSetFromKeys(directKeys) } + { // load indirect set + var err error + p.indirPin, err = loadIndirPin(d, indirectPinDatastoreKey) + if err != nil { + return nil, err + } + } + + // assign services p.dserv = dserv p.dstore = d @@ -177,43 +188,43 @@ func LoadPinner(d ds.Datastore, dserv *mdag.DAGService) (Pinner, error) { func (p *pinner) Flush() error { p.lock.RLock() defer p.lock.RUnlock() - buf := new(bytes.Buffer) - enc := json.NewEncoder(buf) - recurse := p.recursePin.GetKeys() - err := enc.Encode(recurse) + err := storeSet(p.dstore, directPinDatastoreKey, p.directPin.GetKeys()) if err != nil { return err } - err = p.dstore.Put(recursePinDatastoreKey, buf.Bytes()) + err = storeSet(p.dstore, recursePinDatastoreKey, p.recursePin.GetKeys()) if err != nil { return err } - buf = new(bytes.Buffer) - enc = json.NewEncoder(buf) - direct := p.directPin.GetKeys() - err = enc.Encode(direct) - if err != nil { - return err - } - - err = p.dstore.Put(directPinDatastoreKey, buf.Bytes()) - if err != nil { - return err - } - - buf = new(bytes.Buffer) - enc = json.NewEncoder(buf) - err = enc.Encode(p.indirPin.refCounts) - if err != nil { - return err - } - - err = p.dstore.Put(indirectPinDatastoreKey, buf.Bytes()) + err = storeIndirPin(p.dstore, indirectPinDatastoreKey, p.indirPin) if err != nil { return err } return nil } + +// helpers to marshal / unmarshal a pin set +func storeSet(d ds.Datastore, k ds.Key, val interface{}) error { + buf, err := json.Marshal(val) + if err != nil { + return err + } + + return d.Put(k, buf) +} + +func loadSet(d ds.Datastore, k ds.Key, val interface{}) error { + buf, err := d.Get(k) + if err != nil { + return err + } + + bf, ok := buf.([]byte) + if !ok { + return errors.New("invalid pin set value in datastore") + } + return json.Unmarshal(bf, val) +} diff --git a/pin/pin_test.go b/pin/pin_test.go index 00c9c6f3c..10b5862b9 100644 --- a/pin/pin_test.go +++ b/pin/pin_test.go @@ -3,7 +3,7 @@ package pin import ( "testing" - "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore" + ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore" bs "github.com/jbenet/go-ipfs/blockservice" mdag "github.com/jbenet/go-ipfs/merkledag" "github.com/jbenet/go-ipfs/util" @@ -18,7 +18,7 @@ func randNode() (*mdag.Node, util.Key) { } func TestPinnerBasic(t *testing.T) { - dstore := datastore.NewMapDatastore() + dstore := ds.NewMapDatastore() bserv, err := bs.NewBlockService(dstore, nil) if err != nil { t.Fatal(err) @@ -103,7 +103,7 @@ func TestPinnerBasic(t *testing.T) { // c should still be pinned under b if !p.IsPinned(ck) { - t.Fatal("Recursive unpin fail.") + t.Fatal("Recursive / indirect unpin fail.") } err = p.Flush()