fixed pin json marshal

blocks/set/set.go

@@ -1,10 +1,6 @@
 package set
 
 import (
-	"errors"
-
-	ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
-
 	"github.com/jbenet/go-ipfs/blocks/bloom"
 	"github.com/jbenet/go-ipfs/util"
 )
@@ -28,19 +24,6 @@ func SimpleSetFromKeys(keys []util.Key) BlockSet {
 	return sbs
 }
 
-func SetFromDatastore(d ds.Datastore, k ds.Key) (BlockSet, error) {
-	ikeys, err := d.Get(k)
-	if err != nil {
-		return nil, err
-	}
-
-	keys, ok := ikeys.([]util.Key)
-	if !ok {
-		return nil, errors.New("Incorrect type for keys from datastore")
-	}
-	return SimpleSetFromKeys(keys), nil
-}
-
 func NewSimpleBlockSet() BlockSet {
 	return &simpleBlockSet{blocks: make(map[util.Key]struct{})}
 }
@@ -64,7 +47,7 @@ func (b *simpleBlockSet) HasKey(k util.Key) bool {
 
 func (b *simpleBlockSet) GetBloomFilter() bloom.Filter {
 	f := bloom.BasicFilter()
-	for k, _ := range b.blocks {
+	for k := range b.blocks {
 		f.Add([]byte(k))
 	}
 	return f
@@ -72,7 +55,7 @@ func (b *simpleBlockSet) GetBloomFilter() bloom.Filter {
 
 func (b *simpleBlockSet) GetKeys() []util.Key {
 	var out []util.Key
-	for k, _ := range b.blocks {
+	for k := range b.blocks {
 		out = append(out, k)
 	}
 	return out

pin/indirect.go

@@ -1,8 +1,6 @@
 package pin
 
 import (
-	"errors"
-
 	ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
 	"github.com/jbenet/go-ipfs/blocks/set"
 	"github.com/jbenet/go-ipfs/util"
@@ -21,23 +19,33 @@ func NewIndirectPin(dstore ds.Datastore) *indirectPin {
 }
 
 func loadIndirPin(d ds.Datastore, k ds.Key) (*indirectPin, error) {
-	irefcnt, err := d.Get(k)
+	var rcStore map[string]int
+	err := loadSet(d, k, &rcStore)
 	if err != nil {
 		return nil, err
 	}
-	refcnt, ok := irefcnt.(map[util.Key]int)
-	if !ok {
-		return nil, errors.New("invalid type from datastore")
-	}
 
+	refcnt := make(map[util.Key]int)
 	var keys []util.Key
-	for k, _ := range refcnt {
+	for encK, v := range rcStore {
+		k := util.B58KeyDecode(encK)
 		keys = append(keys, k)
+		refcnt[k] = v
 	}
+	log.Debug("indirPin keys: %#v", keys)
 
 	return &indirectPin{blockset: set.SimpleSetFromKeys(keys), refCounts: refcnt}, nil
 }
 
+func storeIndirPin(d ds.Datastore, k ds.Key, p *indirectPin) error {
+
+	rcStore := map[string]int{}
+	for k, v := range p.refCounts {
+		rcStore[util.B58KeyEncode(k)] = v
+	}
+	return storeSet(d, k, rcStore)
+}
+
 func (i *indirectPin) Increment(k util.Key) {
 	c := i.refCounts[k]
 	i.refCounts[k] = c + 1

pin/pin.go

@@ -3,8 +3,9 @@ package pin
 import (
 
 	//ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go"
-	"bytes"
+
 	"encoding/json"
+	"errors"
 	"sync"
 
 	ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
@@ -14,6 +15,7 @@ import (
 	"github.com/jbenet/go-ipfs/util"
 )
 
+var log = util.Logger("pin")
 var recursePinDatastoreKey = ds.NewKey("/local/pins/recursive/keys")
 var directPinDatastoreKey = ds.NewKey("/local/pins/direct/keys")
 var indirectPinDatastoreKey = ds.NewKey("/local/pins/indirect/keys")
@@ -89,9 +91,8 @@ func (p *pinner) Unpin(k util.Key, recurse bool) error {
 		}
 
 		return p.unpinLinks(node)
-	} else {
-		p.directPin.RemoveBlock(k)
 	}
+	p.directPin.RemoveBlock(k)
 	return nil
 }
 
@@ -153,21 +154,31 @@ func (p *pinner) IsPinned(key util.Key) bool {
 func LoadPinner(d ds.Datastore, dserv *mdag.DAGService) (Pinner, error) {
 	p := new(pinner)
 
-	var err error
+	{ // load recursive set
-	p.recursePin, err = set.SetFromDatastore(d, recursePinDatastoreKey)
+		var recurseKeys []util.Key
-	if err != nil {
+		if err := loadSet(d, recursePinDatastoreKey, &recurseKeys); err != nil {
 			return nil, err
 		}
-	p.directPin, err = set.SetFromDatastore(d, directPinDatastoreKey)
+		p.recursePin = set.SimpleSetFromKeys(recurseKeys)
-	if err != nil {
-		return nil, err
 	}
 
+	{ // load direct set
+		var directKeys []util.Key
+		if err := loadSet(d, directPinDatastoreKey, &directKeys); err != nil {
+			return nil, err
+		}
+		p.directPin = set.SimpleSetFromKeys(directKeys)
+	}
+
+	{ // load indirect set
+		var err error
 		p.indirPin, err = loadIndirPin(d, indirectPinDatastoreKey)
 		if err != nil {
 			return nil, err
 		}
+	}
 
+	// assign services
 	p.dserv = dserv
 	p.dstore = d
 
@@ -177,43 +188,43 @@ func LoadPinner(d ds.Datastore, dserv *mdag.DAGService) (Pinner, error) {
 func (p *pinner) Flush() error {
 	p.lock.RLock()
 	defer p.lock.RUnlock()
-	buf := new(bytes.Buffer)
-	enc := json.NewEncoder(buf)
 
-	recurse := p.recursePin.GetKeys()
+	err := storeSet(p.dstore, directPinDatastoreKey, p.directPin.GetKeys())
-	err := enc.Encode(recurse)
 	if err != nil {
 		return err
 	}
 
-	err = p.dstore.Put(recursePinDatastoreKey, buf.Bytes())
+	err = storeSet(p.dstore, recursePinDatastoreKey, p.recursePin.GetKeys())
 	if err != nil {
 		return err
 	}
 
-	buf = new(bytes.Buffer)
+	err = storeIndirPin(p.dstore, indirectPinDatastoreKey, p.indirPin)
-	enc = json.NewEncoder(buf)
-	direct := p.directPin.GetKeys()
-	err = enc.Encode(direct)
-	if err != nil {
-		return err
-	}
-
-	err = p.dstore.Put(directPinDatastoreKey, buf.Bytes())
-	if err != nil {
-		return err
-	}
-
-	buf = new(bytes.Buffer)
-	enc = json.NewEncoder(buf)
-	err = enc.Encode(p.indirPin.refCounts)
-	if err != nil {
-		return err
-	}
-
-	err = p.dstore.Put(indirectPinDatastoreKey, buf.Bytes())
 	if err != nil {
 		return err
 	}
 	return nil
 }
+
+// helpers to marshal / unmarshal a pin set
+func storeSet(d ds.Datastore, k ds.Key, val interface{}) error {
+	buf, err := json.Marshal(val)
+	if err != nil {
+		return err
+	}
+
+	return d.Put(k, buf)
+}
+
+func loadSet(d ds.Datastore, k ds.Key, val interface{}) error {
+	buf, err := d.Get(k)
+	if err != nil {
+		return err
+	}
+
+	bf, ok := buf.([]byte)
+	if !ok {
+		return errors.New("invalid pin set value in datastore")
+	}
+	return json.Unmarshal(bf, val)
+}

pin/pin_test.go

@@ -3,7 +3,7 @@ package pin
 import (
 	"testing"
 
-	"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
+	ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
 	bs "github.com/jbenet/go-ipfs/blockservice"
 	mdag "github.com/jbenet/go-ipfs/merkledag"
 	"github.com/jbenet/go-ipfs/util"
@@ -18,7 +18,7 @@ func randNode() (*mdag.Node, util.Key) {
 }
 
 func TestPinnerBasic(t *testing.T) {
-	dstore := datastore.NewMapDatastore()
+	dstore := ds.NewMapDatastore()
 	bserv, err := bs.NewBlockService(dstore, nil)
 	if err != nil {
 		t.Fatal(err)
@@ -103,7 +103,7 @@ func TestPinnerBasic(t *testing.T) {
 
 	// c should still be pinned under b
 	if !p.IsPinned(ck) {
-		t.Fatal("Recursive unpin fail.")
+		t.Fatal("Recursive / indirect unpin fail.")
 	}
 
 	err = p.Flush()