cache ipns entries to speed things up a little

License: MIT
Signed-off-by: Jeromy <jeromyj@gmail.com>

core/commands/ipns.go

@@ -45,6 +45,7 @@ Resolve the value of another name:
 	},
 	Options: []cmds.Option{
 		cmds.BoolOption("recursive", "r", "Resolve until the result is not an IPNS name"),
+		cmds.BoolOption("nocache", "n", "Do not used cached entries"),
 	},
 	Run: func(req cmds.Request, res cmds.Response) {
 
@@ -62,13 +63,27 @@ Resolve the value of another name:
 			}
 		}
 
-		router := n.Routing
+		nocache, _, _ := req.Option("nocache").Bool()
-		if local, _, _ := req.Option("local").Bool(); local {
+		local, _, _ := req.Option("local").Bool()
-			router = offline.NewOfflineRouter(n.Repo.Datastore(), n.PrivateKey)
+
+		// default to nodes namesys resolver
+		var resolver namesys.Resolver = n.Namesys
+
+		if local && nocache {
+			res.SetError(errors.New("cannot specify both local and nocache"), cmds.ErrNormal)
+			return
+		}
+
+		if local {
+			offroute := offline.NewOfflineRouter(n.Repo.Datastore(), n.PrivateKey)
+			resolver = namesys.NewRoutingResolver(offroute, 0)
+		}
+
+		if nocache {
+			resolver = namesys.NewNameSystem(n.Routing, n.Repo.Datastore(), 0)
 		}
 
 		var name string
-
 		if len(req.Arguments()) == 0 {
 			if n.Identity == "" {
 				res.SetError(errors.New("Identity not loaded!"), cmds.ErrNormal)
@@ -86,7 +101,10 @@ Resolve the value of another name:
 			depth = namesys.DefaultDepthLimit
 		}
 
-		resolver := namesys.NewRoutingResolver(router)
+		if !strings.HasPrefix(name, "/ipns/") {
+			name = "/ipns/" + name
+		}
+
 		output, err := resolver.ResolveN(req.Context(), name, depth)
 		if err != nil {
 			res.SetError(err, cmds.ErrNormal)

core/commands/publish.go

@@ -52,6 +52,7 @@ Publish an <ipfs-path> to another public key (not implemented):
 	Options: []cmds.Option{
 		cmds.BoolOption("resolve", "resolve given path before publishing (default=true)"),
 		cmds.StringOption("lifetime", "t", "time duration that the record will be valid for (default: 24hrs)"),
+		cmds.StringOption("ttl", "time duration this record should be cached for (caution: experimental)"),
 	},
 	Run: func(req cmds.Request, res cmds.Response) {
 		log.Debug("Begin Publish")
@@ -96,7 +97,18 @@ Publish an <ipfs-path> to another public key (not implemented):
 			popts.pubValidTime = d
 		}
 
-		output, err := publish(req.Context(), n, n.PrivateKey, path.Path(pstr), popts)
+		ctx := req.Context()
+		if ttl, found, _ := req.Option("ttl").String(); found {
+			d, err := time.ParseDuration(ttl)
+			if err != nil {
+				res.SetError(err, cmds.ErrNormal)
+				return
+			}
+
+			ctx = context.WithValue(ctx, "ipns-publish-ttl", d)
+		}
+
+		output, err := publish(ctx, n, n.PrivateKey, path.Path(pstr), popts)
 		if err != nil {
 			res.SetError(err, cmds.ErrNormal)
 			return

core/core.go

@@ -226,8 +226,13 @@ func (n *IpfsNode) startOnlineServicesWithHost(ctx context.Context, host p2phost
 	bitswapNetwork := bsnet.NewFromIpfsHost(n.PeerHost, n.Routing)
 	n.Exchange = bitswap.New(ctx, n.Identity, bitswapNetwork, n.Blockstore, alwaysSendToPeer)
 
+	size, err := n.getCacheSize()
+	if err != nil {
+		return err
+	}
+
 	// setup name system
-	n.Namesys = namesys.NewNameSystem(n.Routing, n.Repo.Datastore())
+	n.Namesys = namesys.NewNameSystem(n.Routing, n.Repo.Datastore(), size)
 
 	// setup ipns republishing
 	err = n.setupIpnsRepublisher()
@@ -238,6 +243,23 @@ func (n *IpfsNode) startOnlineServicesWithHost(ctx context.Context, host p2phost
 	return nil
 }
 
+// getCacheSize returns cache life and cache size
+func (n *IpfsNode) getCacheSize() (int, error) {
+	cfg, err := n.Repo.Config()
+	if err != nil {
+		return 0, err
+	}
+
+	cs := cfg.Ipns.ResolveCacheSize
+	if cs == 0 {
+		cs = 128
+	}
+	if cs < 0 {
+		return 0, fmt.Errorf("cannot specify negative resolve cache size")
+	}
+	return cs, nil
+}
+
 func (n *IpfsNode) setupIpnsRepublisher() error {
 	cfg, err := n.Repo.Config()
 	if err != nil {
@@ -456,7 +478,12 @@ func (n *IpfsNode) SetupOfflineRouting() error {
 
 	n.Routing = offroute.NewOfflineRouter(n.Repo.Datastore(), n.PrivateKey)
 
-	n.Namesys = namesys.NewNameSystem(n.Routing, n.Repo.Datastore())
+	size, err := n.getCacheSize()
+	if err != nil {
+		return err
+	}
+
+	n.Namesys = namesys.NewNameSystem(n.Routing, n.Repo.Datastore(), size)
 
 	return nil
 }

fuse/ipns/ipns_test.go

@@ -113,7 +113,7 @@ func setupIpnsTest(t *testing.T, node *core.IpfsNode) (*core.IpfsNode, *fstest.M
 		}
 
 		node.Routing = offroute.NewOfflineRouter(node.Repo.Datastore(), node.PrivateKey)
-		node.Namesys = namesys.NewNameSystem(node.Routing, node.Repo.Datastore())
+		node.Namesys = namesys.NewNameSystem(node.Routing, node.Repo.Datastore(), 0)
 
 		ipnsfs, err := nsfs.NewFilesystem(context.Background(), node.DAG, node.Namesys, node.Pinning, node.PrivateKey)
 		if err != nil {

namesys/namesys.go

@@ -26,12 +26,12 @@ type mpns struct {
 }
 
 // NewNameSystem will construct the IPFS naming system based on Routing
-func NewNameSystem(r routing.IpfsRouting, ds ds.Datastore) NameSystem {
+func NewNameSystem(r routing.IpfsRouting, ds ds.Datastore, cachesize int) NameSystem {
 	return &mpns{
 		resolvers: map[string]resolver{
 			"dns":      newDNSResolver(),
 			"proquint": new(ProquintResolver),
-			"dht":      newRoutingResolver(r),
+			"dht":      NewRoutingResolver(r, cachesize),
 		},
 		publishers: map[string]Publisher{
 			"/ipns/": NewRoutingPublisher(r, ds),
@@ -39,6 +39,8 @@ func NewNameSystem(r routing.IpfsRouting, ds ds.Datastore) NameSystem {
 	}
 }
 
+const DefaultResolverCacheTTL = time.Minute
+
 // Resolve implements Resolver.
 func (ns *mpns) Resolve(ctx context.Context, name string) (path.Path, error) {
 	return ns.ResolveN(ctx, name, DefaultDepthLimit)

namesys/pb/namesys.pb.go

@@ -57,6 +57,7 @@ type IpnsEntry struct {
 	ValidityType     *IpnsEntry_ValidityType `protobuf:"varint,3,opt,name=validityType,enum=namesys.pb.IpnsEntry_ValidityType" json:"validityType,omitempty"`
 	Validity         []byte                  `protobuf:"bytes,4,opt,name=validity" json:"validity,omitempty"`
 	Sequence         *uint64                 `protobuf:"varint,5,opt,name=sequence" json:"sequence,omitempty"`
+	Ttl              *uint64                 `protobuf:"varint,6,opt,name=ttl" json:"ttl,omitempty"`
 	XXX_unrecognized []byte                  `json:"-"`
 }
 
@@ -99,6 +100,13 @@ func (m *IpnsEntry) GetSequence() uint64 {
 	return 0
 }
 
+func (m *IpnsEntry) GetTtl() uint64 {
+	if m != nil && m.Ttl != nil {
+		return *m.Ttl
+	}
+	return 0
+}
+
 func init() {
 	proto.RegisterEnum("namesys.pb.IpnsEntry_ValidityType", IpnsEntry_ValidityType_name, IpnsEntry_ValidityType_value)
 }

namesys/pb/namesys.proto

@@ -12,4 +12,6 @@ message IpnsEntry {
 	optional bytes validity = 4;
 
 	optional uint64 sequence = 5;
+
+	optional uint64 ttl = 6;
 }

namesys/publisher.go

@@ -121,6 +121,19 @@ func (p *ipnsPublisher) getPreviousSeqNo(ctx context.Context, ipnskey key.Key) (
 	return e.GetSequence(), nil
 }
 
+// setting the TTL on published records is an experimental feature.
+// as such, i'm using the context to wire it through to avoid changing too
+// much code along the way.
+func checkCtxTTL(ctx context.Context) (time.Duration, bool) {
+	v := ctx.Value("ipns-publish-ttl")
+	if v == nil {
+		return 0, false
+	}
+
+	d, ok := v.(time.Duration)
+	return d, ok
+}
+
 func PutRecordToRouting(ctx context.Context, k ci.PrivKey, value path.Path, seqnum uint64, eol time.Time, r routing.IpfsRouting, id peer.ID) error {
 	namekey, ipnskey := IpnsKeysForID(id)
 	entry, err := CreateRoutingEntryData(k, value, seqnum, eol)
@@ -128,6 +141,11 @@ func PutRecordToRouting(ctx context.Context, k ci.PrivKey, value path.Path, seqn
 		return err
 	}
 
+	ttl, ok := checkCtxTTL(ctx)
+	if ok {
+		entry.Ttl = proto.Uint64(uint64(ttl.Nanoseconds()))
+	}
+
 	err = PublishEntry(ctx, r, ipnskey, entry)
 	if err != nil {
 		return err

namesys/republisher/repub_test.go

@@ -18,6 +18,8 @@ import (
 )
 
 func TestRepublish(t *testing.T) {
+	// set cache life to zero for testing low-period repubs
+
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
@@ -34,6 +36,8 @@ func TestRepublish(t *testing.T) {
 			t.Fatal(err)
 		}
 
+		nd.Namesys = namesys.NewNameSystem(nd.Routing, nd.Repo.Datastore(), 0)
+
 		nodes = append(nodes, nd)
 	}
 

namesys/resolve_test.go

@@ -19,7 +19,7 @@ func TestRoutingResolve(t *testing.T) {
 	d := mockrouting.NewServer().Client(testutil.RandIdentityOrFatal(t))
 	dstore := ds.NewMapDatastore()
 
-	resolver := NewRoutingResolver(d)
+	resolver := NewRoutingResolver(d, 0)
 	publisher := NewRoutingPublisher(d, dstore)
 
 	privk, pubk, err := testutil.RandTestKeyPair(512)
@@ -53,7 +53,7 @@ func TestPrexistingExpiredRecord(t *testing.T) {
 	dstore := ds.NewMapDatastore()
 	d := mockrouting.NewServer().ClientWithDatastore(context.Background(), testutil.RandIdentityOrFatal(t), dstore)
 
-	resolver := NewRoutingResolver(d)
+	resolver := NewRoutingResolver(d, 0)
 	publisher := NewRoutingPublisher(d, dstore)
 
 	privk, pubk, err := testutil.RandTestKeyPair(512)
@@ -90,7 +90,7 @@ func TestPrexistingRecord(t *testing.T) {
 	dstore := ds.NewMapDatastore()
 	d := mockrouting.NewServer().ClientWithDatastore(context.Background(), testutil.RandIdentityOrFatal(t), dstore)
 
-	resolver := NewRoutingResolver(d)
+	resolver := NewRoutingResolver(d, 0)
 	publisher := NewRoutingPublisher(d, dstore)
 
 	privk, pubk, err := testutil.RandTestKeyPair(512)

namesys/routing.go

@@ -2,16 +2,19 @@ package namesys
 
 import (
 	"fmt"
+	"time"
 
 	proto "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/gogo/protobuf/proto"
+	lru "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/hashicorp/golang-lru"
 	mh "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multihash"
 	"github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
+	logging "github.com/ipfs/go-ipfs/vendor/QmTBXYb6y2ZcJmoXVKk3pf9rzSEjbCg7tQaJW7RSuH14nv/go-log"
 
 	key "github.com/ipfs/go-ipfs/blocks/key"
 	pb "github.com/ipfs/go-ipfs/namesys/pb"
 	path "github.com/ipfs/go-ipfs/path"
 	routing "github.com/ipfs/go-ipfs/routing"
-	logging "github.com/ipfs/go-ipfs/vendor/QmTBXYb6y2ZcJmoXVKk3pf9rzSEjbCg7tQaJW7RSuH14nv/go-log"
+	u "github.com/ipfs/go-ipfs/util"
 )
 
 var log = logging.Logger("namesys")
@@ -19,25 +22,84 @@ var log = logging.Logger("namesys")
 // routingResolver implements NSResolver for the main IPFS SFS-like naming
 type routingResolver struct {
 	routing routing.IpfsRouting
+
+	cache *lru.Cache
+}
+
+func (r *routingResolver) cacheGet(name string) (path.Path, bool) {
+	if r.cache == nil {
+		return "", false
+	}
+
+	ientry, ok := r.cache.Get(name)
+	if !ok {
+		return "", false
+	}
+
+	entry, ok := ientry.(cacheEntry)
+	if !ok {
+		// should never happen, purely for sanity
+		log.Panicf("unexpected type %T in cache for %q.", ientry, name)
+	}
+
+	if time.Now().Before(entry.eol) {
+		return entry.val, true
+	}
+
+	r.cache.Remove(name)
+
+	return "", false
+}
+
+func (r *routingResolver) cacheSet(name string, val path.Path, rec *pb.IpnsEntry) {
+	if r.cache == nil {
+		return
+	}
+
+	// if completely unspecified, just use one minute
+	ttl := DefaultResolverCacheTTL
+	if rec.Ttl != nil {
+		recttl := time.Duration(rec.GetTtl())
+		if recttl >= 0 {
+			ttl = recttl
+		}
+	}
+
+	cacheTil := time.Now().Add(ttl)
+	eol, ok := checkEOL(rec)
+	if ok && eol.Before(cacheTil) {
+		cacheTil = eol
+	}
+
+	r.cache.Add(name, cacheEntry{
+		val: val,
+		eol: cacheTil,
+	})
+}
+
+type cacheEntry struct {
+	val path.Path
+	eol time.Time
 }
 
 // NewRoutingResolver constructs a name resolver using the IPFS Routing system
 // to implement SFS-like naming on top.
-func NewRoutingResolver(route routing.IpfsRouting) Resolver {
+// cachesize is the limit of the number of entries in the lru cache. Setting it
+// to '0' will disable caching.
+func NewRoutingResolver(route routing.IpfsRouting, cachesize int) *routingResolver {
 	if route == nil {
 		panic("attempt to create resolver with nil routing system")
 	}
 
-	return &routingResolver{routing: route}
+	var cache *lru.Cache
-}
+	if cachesize > 0 {
-
+		cache, _ = lru.New(cachesize)
-// newRoutingResolver returns a resolver instead of a Resolver.
-func newRoutingResolver(route routing.IpfsRouting) resolver {
-	if route == nil {
-		panic("attempt to create resolver with nil routing system")
 	}
 
-	return &routingResolver{routing: route}
+	return &routingResolver{
+		routing: route,
+		cache:   cache,
+	}
 }
 
 // Resolve implements Resolver.
@@ -54,6 +116,11 @@ func (r *routingResolver) ResolveN(ctx context.Context, name string, depth int)
 // resolve SFS-like names.
 func (r *routingResolver) resolveOnce(ctx context.Context, name string) (path.Path, error) {
 	log.Debugf("RoutingResolve: '%s'", name)
+	cached, ok := r.cacheGet(name)
+	if ok {
+		return cached, nil
+	}
+
 	hash, err := mh.FromB58String(name)
 	if err != nil {
 		log.Warning("RoutingResolve: bad input hash: [%s]\n", name)
@@ -98,10 +165,29 @@ func (r *routingResolver) resolveOnce(ctx context.Context, name string) (path.Pa
 	valh, err := mh.Cast(entry.GetValue())
 	if err != nil {
 		// Not a multihash, probably a new record
-		return path.ParsePath(string(entry.GetValue()))
+		p, err := path.ParsePath(string(entry.GetValue()))
+		if err != nil {
+			return "", err
+		}
+
+		r.cacheSet(name, p, entry)
+		return p, nil
 	} else {
 		// Its an old style multihash record
 		log.Warning("Detected old style multihash record")
-		return path.FromKey(key.Key(valh)), nil
+		p := path.FromKey(key.Key(valh))
+		r.cacheSet(name, p, entry)
+		return p, nil
 	}
 }
+
+func checkEOL(e *pb.IpnsEntry) (time.Time, bool) {
+	if e.GetValidityType() == pb.IpnsEntry_EOL {
+		eol, err := u.ParseRFC3339(string(e.GetValidity()))
+		if err != nil {
+			return time.Time{}, false
+		}
+		return eol, true
+	}
+	return time.Time{}, false
+}

repo/config/init.go

@@ -64,6 +64,10 @@ func Init(out io.Writer, nBitsForKeypair int) (*Config, error) {
 			IPNS: "/ipns",
 		},
 
+		Ipns: Ipns{
+			ResolveCacheSize: 128,
+		},
+
 		// tracking ipfs version used to generate the init folder and adding
 		// update checker default setting.
 		Version: VersionDefaultValue(),

repo/config/ipns.go

@@ -3,4 +3,6 @@ package config
 type Ipns struct {
 	RepublishPeriod string
 	RecordLifetime  string
+
+	ResolveCacheSize int
 }

test/sharness/t0240-republisher.sh

@@ -26,6 +26,7 @@ setup_iptb() {
 		for i in $(test_seq 0 3)
 		do
 			ipfsi $i config Ipns.RepublishPeriod 20s
+			ipfsi $i config --json Ipns.ResolveCacheSize 0
 		done
 	'