add blockset and bloomfilter and beginnings of pinning service

blocks/bloom/filter.go

@@ -0,0 +1,70 @@
+package bloom
+
+import (
+	"fmt"
+	"hash"
+	"hash/adler32"
+	"hash/crc32"
+	"hash/fnv"
+	"math/big"
+)
+
+type Filter interface {
+	Add([]byte)
+	Find([]byte) bool
+}
+
+func BasicFilter() Filter {
+	// Non crypto hashes, because speed
+	return NewFilter(2048, adler32.New(), fnv.New32(), crc32.NewIEEE())
+}
+
+func NewFilter(size int, hashes ...hash.Hash) Filter {
+	return &filter{
+		filter: make([]byte, size),
+		hashes: hashes,
+	}
+}
+
+type filter struct {
+	filter []byte
+	hashes []hash.Hash
+}
+
+func (f *filter) Add(k []byte) {
+	for _, h := range f.hashes {
+		i := bytesMod(h.Sum(k), int64(len(f.filter)*8))
+		f.setBit(i)
+	}
+}
+
+func (f *filter) Find(k []byte) bool {
+	for _, h := range f.hashes {
+		i := bytesMod(h.Sum(k), int64(len(f.filter)*8))
+		if !f.getBit(i) {
+			return false
+		}
+	}
+	return true
+}
+
+func (f *filter) setBit(i int64) {
+	fmt.Printf("setting bit %d\n", i)
+	f.filter[i/8] |= (1 << byte(i%8))
+}
+
+func (f *filter) getBit(i int64) bool {
+	fmt.Printf("getting bit %d\n", i)
+	return f.filter[i/8]&(1<<byte(i%8)) != 0
+}
+
+func bytesMod(b []byte, modulo int64) int64 {
+	i := big.NewInt(0)
+	i = i.SetBytes(b)
+
+	bigmod := big.NewInt(int64(modulo))
+	result := big.NewInt(0)
+	result.Mod(i, bigmod)
+
+	return result.Int64()
+}

blocks/bloom/filter.proto

@@ -0,0 +1,10 @@
+package bloom;
+
+message PackedFilter {
+	enum HashType {
+
+	}
+	optional bool compressed;
+	optional bytes data;
+	repeated HashType hashes;
+}

blocks/bloom/filter_test.go

@@ -0,0 +1,30 @@
+package bloom
+
+import "testing"
+
+func TestFilter(t *testing.T) {
+	f := BasicFilter()
+	keys := [][]byte{
+		[]byte("hello"),
+		[]byte("fish"),
+		[]byte("ipfsrocks"),
+	}
+
+	f.Add(keys[0])
+	if !f.Find(keys[0]) {
+		t.Fatal("Failed to find single inserted key!")
+	}
+
+	f.Add(keys[1])
+	if !f.Find(keys[1]) {
+		t.Fatal("Failed to find key!")
+	}
+
+	f.Add(keys[2])
+
+	for _, k := range keys {
+		if !f.Find(k) {
+			t.Fatal("Couldnt find one of three keys")
+		}
+	}
+}

blocks/set/set.go

@@ -0,0 +1,42 @@
+package set
+
+import (
+	"github.com/jbenet/go-ipfs/blocks/bloom"
+	"github.com/jbenet/go-ipfs/util"
+)
+
+type BlockSet interface {
+	AddBlock(util.Key)
+	RemoveBlock(util.Key)
+	HasKey(util.Key) bool
+	GetBloomFilter() bloom.Filter
+}
+
+func NewSimpleBlockSet() BlockSet {
+	return &simpleBlockSet{blocks: make(map[util.Key]struct{})}
+}
+
+type simpleBlockSet struct {
+	blocks map[util.Key]struct{}
+}
+
+func (b *simpleBlockSet) AddBlock(k util.Key) {
+	b.blocks[k] = struct{}{}
+}
+
+func (b *simpleBlockSet) RemoveBlock(k util.Key) {
+	delete(b.blocks, k)
+}
+
+func (b *simpleBlockSet) HasKey(k util.Key) bool {
+	_, has := b.blocks[k]
+	return has
+}
+
+func (b *simpleBlockSet) GetBloomFilter() bloom.Filter {
+	f := bloom.BasicFilter()
+	for k, _ := range b.blocks {
+		f.Add([]byte(k))
+	}
+	return f
+}

merkledag/merkledag.go

@@ -59,6 +59,14 @@ func MakeLink(n *Node) (*Link, error) {
 	}, nil
 }
 
+func (l *Link) GetNode(serv *DAGService) (*Node, error) {
+	if l.Node != nil {
+		return l.Node, nil
+	}
+
+	return serv.Get(u.Key(l.Hash))
+}
+
 // AddNodeLink adds a link to another node.
 func (n *Node) AddNodeLink(name string, that *Node) error {
 	lnk, err := MakeLink(that)

pin/pin.go

@@ -0,0 +1,88 @@
+package pin
+
+import (
+	ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go"
+	"github.com/jbenet/go-ipfs/blocks/set"
+	mdag "github.com/jbenet/go-ipfs/merkledag"
+	"github.com/jbenet/go-ipfs/util"
+)
+
+type Pinner interface {
+	Pin(*mdag.Node, bool) error
+	Unpin(util.Key, bool) error
+}
+
+type pinner struct {
+	recursePin set.BlockSet
+	directPin  set.BlockSet
+	indirPin   set.BlockSet
+	dserv      *mdag.DAGService
+}
+
+func NewPinner(dstore ds.Datastore, serv *mdag.DAGService) Pinner {
+	return &pinner{
+		recursePin: set.NewSimpleBlockSet(),
+		directPin:  set.NewSimpleBlockSet(),
+		indirPin:   NewRefCountBlockSet(),
+		dserv:      serv,
+	}
+}
+
+func (p *pinner) Pin(node *mdag.Node, recurse bool) error {
+	k, err := node.Key()
+	if err != nil {
+		return err
+	}
+
+	if recurse {
+		if p.recursePin.HasKey(k) {
+			return nil
+		}
+
+		p.recursePin.AddBlock(k)
+
+		err := p.pinLinks(node)
+		if err != nil {
+			return err
+		}
+	} else {
+		p.directPin.AddBlock(k)
+	}
+	return nil
+}
+
+func (p *pinner) Unpin(k util.Key, recurse bool) error {
+	panic("not yet implemented!")
+	return nil
+}
+
+func (p *pinner) pinIndirectRecurse(node *mdag.Node) error {
+	k, err := node.Key()
+	if err != nil {
+		return err
+	}
+
+	p.indirPin.AddBlock(k)
+	return p.pinLinks(node)
+}
+
+func (p *pinner) pinLinks(node *mdag.Node) error {
+	for _, l := range node.Links {
+		subnode, err := l.GetNode(p.dserv)
+		if err != nil {
+			// TODO: Maybe just log and continue?
+			return err
+		}
+		err = p.pinIndirectRecurse(subnode)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (p *pinner) IsPinned(key util.Key) bool {
+	return p.recursePin.HasKey(key) ||
+		p.directPin.HasKey(key) ||
+		p.indirPin.HasKey(key)
+}

pin/refset.go

@@ -0,0 +1,40 @@
+package pin
+
+import (
+	"github.com/jbenet/go-ipfs/blocks/bloom"
+	"github.com/jbenet/go-ipfs/blocks/set"
+	"github.com/jbenet/go-ipfs/util"
+)
+
+type refCntBlockSet struct {
+	blocks map[util.Key]int
+}
+
+func NewRefCountBlockSet() set.BlockSet {
+	return &refCntBlockSet{blocks: make(map[util.Key]int)}
+}
+
+func (r *refCntBlockSet) AddBlock(k util.Key) {
+	r.blocks[k]++
+}
+
+func (r *refCntBlockSet) RemoveBlock(k util.Key) {
+	v, ok := r.blocks[k]
+	if !ok {
+		return
+	}
+	if v <= 1 {
+		delete(r.blocks, k)
+	} else {
+		r.blocks[k] = v - 1
+	}
+}
+
+func (r *refCntBlockSet) HasKey(k util.Key) bool {
+	_, ok := r.blocks[k]
+	return ok
+}
+
+func (r *refCntBlockSet) GetBloomFilter() bloom.Filter {
+	return nil
+}