opensource/kubo
1
0
Fork 0
mirror of https://github.com/ipfs/kubo.git synced 2025-07-01 02:30:39 +08:00
Code Issues Projects Releases Wiki Activity

fixed pin json marshal

Browse Source
This commit is contained in:
Juan Batiz-Benet
2014-10-22 02:55:22 -07:00
parent 5d86ce56ac
commit 4b01ad0b7c
4 changed files with 72 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
blocks/set/set.go
View File

@ -1,10 +1,6 @@
package set
import (
"errors"
ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
"github.com/jbenet/go-ipfs/blocks/bloom"
"github.com/jbenet/go-ipfs/util"
)
@ -28,19 +24,6 @@ func SimpleSetFromKeys(keys []util.Key) BlockSet {
return sbs
}
func SetFromDatastore(d ds.Datastore, k ds.Key) (BlockSet, error) {
ikeys, err := d.Get(k)
if err != nil {
return nil, err
}
keys, ok := ikeys.([]util.Key)
if !ok {
return nil, errors.New("Incorrect type for keys from datastore")
}
return SimpleSetFromKeys(keys), nil
}
func NewSimpleBlockSet() BlockSet {
return &simpleBlockSet{blocks: make(map[util.Key]struct{})}
}
@ -64,7 +47,7 @@ func (b *simpleBlockSet) HasKey(k util.Key) bool {
func (b *simpleBlockSet) GetBloomFilter() bloom.Filter {
f := bloom.BasicFilter()
for k, _ := range b.blocks {
for k := range b.blocks {
f.Add([]byte(k))
}
return f
@ -72,7 +55,7 @@ func (b *simpleBlockSet) GetBloomFilter() bloom.Filter {
func (b *simpleBlockSet) GetKeys() []util.Key {
var out []util.Key
for k, _ := range b.blocks {
for k := range b.blocks {
out = append(out, k)
}
return out

24
pin/indirect.go
View File

@ -1,8 +1,6 @@
package pin
import (
"errors"
ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
"github.com/jbenet/go-ipfs/blocks/set"
"github.com/jbenet/go-ipfs/util"
@ -21,23 +19,33 @@ func NewIndirectPin(dstore ds.Datastore) *indirectPin {
}
func loadIndirPin(d ds.Datastore, k ds.Key) (*indirectPin, error) {
irefcnt, err := d.Get(k)
var rcStore map[string]int
err := loadSet(d, k, &rcStore)
if err != nil {
return nil, err
}
refcnt, ok := irefcnt.(map[util.Key]int)
if !ok {
return nil, errors.New("invalid type from datastore")
}
refcnt := make(map[util.Key]int)
var keys []util.Key
for k, _ := range refcnt {
for encK, v := range rcStore {
k := util.B58KeyDecode(encK)
keys = append(keys, k)
refcnt[k] = v
}
log.Debug("indirPin keys: %#v", keys)
return &indirectPin{blockset: set.SimpleSetFromKeys(keys), refCounts: refcnt}, nil
}
func storeIndirPin(d ds.Datastore, k ds.Key, p *indirectPin) error {
rcStore := map[string]int{}
for k, v := range p.refCounts {
rcStore[util.B58KeyEncode(k)] = v
}
return storeSet(d, k, rcStore)
}
func (i *indirectPin) Increment(k util.Key) {
c := i.refCounts[k]
i.refCounts[k] = c + 1

91
pin/pin.go
View File

@ -3,8 +3,9 @@ package pin
import (
//ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go"
"bytes"
"encoding/json"
"errors"
"sync"
ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
@ -14,6 +15,7 @@ import (
"github.com/jbenet/go-ipfs/util"
)
var log = util.Logger("pin")
var recursePinDatastoreKey = ds.NewKey("/local/pins/recursive/keys")
var directPinDatastoreKey = ds.NewKey("/local/pins/direct/keys")
var indirectPinDatastoreKey = ds.NewKey("/local/pins/indirect/keys")
@ -89,9 +91,8 @@ func (p *pinner) Unpin(k util.Key, recurse bool) error {
}
return p.unpinLinks(node)
} else {
p.directPin.RemoveBlock(k)
}
p.directPin.RemoveBlock(k)
return nil
}
@ -153,21 +154,31 @@ func (p *pinner) IsPinned(key util.Key) bool {
func LoadPinner(d ds.Datastore, dserv *mdag.DAGService) (Pinner, error) {
p := new(pinner)
var err error
p.recursePin, err = set.SetFromDatastore(d, recursePinDatastoreKey)
if err != nil {
return nil, err
}
p.directPin, err = set.SetFromDatastore(d, directPinDatastoreKey)
if err != nil {
return nil, err
{ // load recursive set
var recurseKeys []util.Key
if err := loadSet(d, recursePinDatastoreKey, &recurseKeys); err != nil {
return nil, err
}
p.recursePin = set.SimpleSetFromKeys(recurseKeys)
}
p.indirPin, err = loadIndirPin(d, indirectPinDatastoreKey)
if err != nil {
return nil, err
{ // load direct set
var directKeys []util.Key
if err := loadSet(d, directPinDatastoreKey, &directKeys); err != nil {
return nil, err
}
p.directPin = set.SimpleSetFromKeys(directKeys)
}
{ // load indirect set
var err error
p.indirPin, err = loadIndirPin(d, indirectPinDatastoreKey)
if err != nil {
return nil, err
}
}
// assign services
p.dserv = dserv
p.dstore = d
@ -177,43 +188,43 @@ func LoadPinner(d ds.Datastore, dserv *mdag.DAGService) (Pinner, error) {
func (p *pinner) Flush() error {
p.lock.RLock()
defer p.lock.RUnlock()
buf := new(bytes.Buffer)
enc := json.NewEncoder(buf)
recurse := p.recursePin.GetKeys()
err := enc.Encode(recurse)
err := storeSet(p.dstore, directPinDatastoreKey, p.directPin.GetKeys())
if err != nil {
return err
}
err = p.dstore.Put(recursePinDatastoreKey, buf.Bytes())
err = storeSet(p.dstore, recursePinDatastoreKey, p.recursePin.GetKeys())
if err != nil {
return err
}
buf = new(bytes.Buffer)
enc = json.NewEncoder(buf)
direct := p.directPin.GetKeys()
err = enc.Encode(direct)
if err != nil {
return err
}
err = p.dstore.Put(directPinDatastoreKey, buf.Bytes())
if err != nil {
return err
}
buf = new(bytes.Buffer)
enc = json.NewEncoder(buf)
err = enc.Encode(p.indirPin.refCounts)
if err != nil {
return err
}
err = p.dstore.Put(indirectPinDatastoreKey, buf.Bytes())
err = storeIndirPin(p.dstore, indirectPinDatastoreKey, p.indirPin)
if err != nil {
return err
}
return nil
}
// helpers to marshal / unmarshal a pin set
func storeSet(d ds.Datastore, k ds.Key, val interface{}) error {
buf, err := json.Marshal(val)
if err != nil {
return err
}
return d.Put(k, buf)
}
func loadSet(d ds.Datastore, k ds.Key, val interface{}) error {
buf, err := d.Get(k)
if err != nil {
return err
}
bf, ok := buf.([]byte)
if !ok {
return errors.New("invalid pin set value in datastore")
}
return json.Unmarshal(bf, val)
}

6
pin/pin_test.go
View File

@ -3,7 +3,7 @@ package pin
import (
"testing"
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
bs "github.com/jbenet/go-ipfs/blockservice"
mdag "github.com/jbenet/go-ipfs/merkledag"
"github.com/jbenet/go-ipfs/util"
@ -18,7 +18,7 @@ func randNode() (*mdag.Node, util.Key) {
}
func TestPinnerBasic(t *testing.T) {
dstore := datastore.NewMapDatastore()
dstore := ds.NewMapDatastore()
bserv, err := bs.NewBlockService(dstore, nil)
if err != nil {
t.Fatal(err)
@ -103,7 +103,7 @@ func TestPinnerBasic(t *testing.T) {
// c should still be pinned under b
if !p.IsPinned(ck) {
t.Fatal("Recursive unpin fail.")
t.Fatal("Recursive / indirect unpin fail.")
}
err = p.Flush()