From c7fb7ce17d20961cae549667fdbda76a2639fe00 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Thu, 22 Oct 2015 16:27:31 -0700 Subject: [PATCH] cache ipns entries to speed things up a little License: MIT Signed-off-by: Jeromy --- core/commands/ipns.go | 28 ++++++-- core/commands/publish.go | 14 +++- core/core.go | 31 +++++++- fuse/ipns/ipns_test.go | 2 +- namesys/namesys.go | 6 +- namesys/pb/namesys.pb.go | 8 +++ namesys/pb/namesys.proto | 2 + namesys/publisher.go | 18 +++++ namesys/republisher/repub_test.go | 4 ++ namesys/resolve_test.go | 6 +- namesys/routing.go | 110 +++++++++++++++++++++++++---- repo/config/init.go | 4 ++ repo/config/ipns.go | 2 + test/sharness/t0240-republisher.sh | 1 + 14 files changed, 210 insertions(+), 26 deletions(-) diff --git a/core/commands/ipns.go b/core/commands/ipns.go index 494ef994e..591286914 100644 --- a/core/commands/ipns.go +++ b/core/commands/ipns.go @@ -45,6 +45,7 @@ Resolve the value of another name: }, Options: []cmds.Option{ cmds.BoolOption("recursive", "r", "Resolve until the result is not an IPNS name"), + cmds.BoolOption("nocache", "n", "Do not used cached entries"), }, Run: func(req cmds.Request, res cmds.Response) { @@ -62,13 +63,27 @@ Resolve the value of another name: } } - router := n.Routing - if local, _, _ := req.Option("local").Bool(); local { - router = offline.NewOfflineRouter(n.Repo.Datastore(), n.PrivateKey) + nocache, _, _ := req.Option("nocache").Bool() + local, _, _ := req.Option("local").Bool() + + // default to nodes namesys resolver + var resolver namesys.Resolver = n.Namesys + + if local && nocache { + res.SetError(errors.New("cannot specify both local and nocache"), cmds.ErrNormal) + return + } + + if local { + offroute := offline.NewOfflineRouter(n.Repo.Datastore(), n.PrivateKey) + resolver = namesys.NewRoutingResolver(offroute, 0) + } + + if nocache { + resolver = namesys.NewNameSystem(n.Routing, n.Repo.Datastore(), 0) } var name string - if len(req.Arguments()) == 0 { if n.Identity == "" { res.SetError(errors.New("Identity not loaded!"), cmds.ErrNormal) @@ -86,7 +101,10 @@ Resolve the value of another name: depth = namesys.DefaultDepthLimit } - resolver := namesys.NewRoutingResolver(router) + if !strings.HasPrefix(name, "/ipns/") { + name = "/ipns/" + name + } + output, err := resolver.ResolveN(req.Context(), name, depth) if err != nil { res.SetError(err, cmds.ErrNormal) diff --git a/core/commands/publish.go b/core/commands/publish.go index 0737a0117..0e539868d 100644 --- a/core/commands/publish.go +++ b/core/commands/publish.go @@ -52,6 +52,7 @@ Publish an to another public key (not implemented): Options: []cmds.Option{ cmds.BoolOption("resolve", "resolve given path before publishing (default=true)"), cmds.StringOption("lifetime", "t", "time duration that the record will be valid for (default: 24hrs)"), + cmds.StringOption("ttl", "time duration this record should be cached for (caution: experimental)"), }, Run: func(req cmds.Request, res cmds.Response) { log.Debug("Begin Publish") @@ -96,7 +97,18 @@ Publish an to another public key (not implemented): popts.pubValidTime = d } - output, err := publish(req.Context(), n, n.PrivateKey, path.Path(pstr), popts) + ctx := req.Context() + if ttl, found, _ := req.Option("ttl").String(); found { + d, err := time.ParseDuration(ttl) + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + + ctx = context.WithValue(ctx, "ipns-publish-ttl", d) + } + + output, err := publish(ctx, n, n.PrivateKey, path.Path(pstr), popts) if err != nil { res.SetError(err, cmds.ErrNormal) return diff --git a/core/core.go b/core/core.go index a9cc776c5..54d6bcddf 100644 --- a/core/core.go +++ b/core/core.go @@ -226,8 +226,13 @@ func (n *IpfsNode) startOnlineServicesWithHost(ctx context.Context, host p2phost bitswapNetwork := bsnet.NewFromIpfsHost(n.PeerHost, n.Routing) n.Exchange = bitswap.New(ctx, n.Identity, bitswapNetwork, n.Blockstore, alwaysSendToPeer) + size, err := n.getCacheSize() + if err != nil { + return err + } + // setup name system - n.Namesys = namesys.NewNameSystem(n.Routing, n.Repo.Datastore()) + n.Namesys = namesys.NewNameSystem(n.Routing, n.Repo.Datastore(), size) // setup ipns republishing err = n.setupIpnsRepublisher() @@ -238,6 +243,23 @@ func (n *IpfsNode) startOnlineServicesWithHost(ctx context.Context, host p2phost return nil } +// getCacheSize returns cache life and cache size +func (n *IpfsNode) getCacheSize() (int, error) { + cfg, err := n.Repo.Config() + if err != nil { + return 0, err + } + + cs := cfg.Ipns.ResolveCacheSize + if cs == 0 { + cs = 128 + } + if cs < 0 { + return 0, fmt.Errorf("cannot specify negative resolve cache size") + } + return cs, nil +} + func (n *IpfsNode) setupIpnsRepublisher() error { cfg, err := n.Repo.Config() if err != nil { @@ -456,7 +478,12 @@ func (n *IpfsNode) SetupOfflineRouting() error { n.Routing = offroute.NewOfflineRouter(n.Repo.Datastore(), n.PrivateKey) - n.Namesys = namesys.NewNameSystem(n.Routing, n.Repo.Datastore()) + size, err := n.getCacheSize() + if err != nil { + return err + } + + n.Namesys = namesys.NewNameSystem(n.Routing, n.Repo.Datastore(), size) return nil } diff --git a/fuse/ipns/ipns_test.go b/fuse/ipns/ipns_test.go index f65dd42fd..fdee57418 100644 --- a/fuse/ipns/ipns_test.go +++ b/fuse/ipns/ipns_test.go @@ -113,7 +113,7 @@ func setupIpnsTest(t *testing.T, node *core.IpfsNode) (*core.IpfsNode, *fstest.M } node.Routing = offroute.NewOfflineRouter(node.Repo.Datastore(), node.PrivateKey) - node.Namesys = namesys.NewNameSystem(node.Routing, node.Repo.Datastore()) + node.Namesys = namesys.NewNameSystem(node.Routing, node.Repo.Datastore(), 0) ipnsfs, err := nsfs.NewFilesystem(context.Background(), node.DAG, node.Namesys, node.Pinning, node.PrivateKey) if err != nil { diff --git a/namesys/namesys.go b/namesys/namesys.go index 12b73218b..c61d3496b 100644 --- a/namesys/namesys.go +++ b/namesys/namesys.go @@ -26,12 +26,12 @@ type mpns struct { } // NewNameSystem will construct the IPFS naming system based on Routing -func NewNameSystem(r routing.IpfsRouting, ds ds.Datastore) NameSystem { +func NewNameSystem(r routing.IpfsRouting, ds ds.Datastore, cachesize int) NameSystem { return &mpns{ resolvers: map[string]resolver{ "dns": newDNSResolver(), "proquint": new(ProquintResolver), - "dht": newRoutingResolver(r), + "dht": NewRoutingResolver(r, cachesize), }, publishers: map[string]Publisher{ "/ipns/": NewRoutingPublisher(r, ds), @@ -39,6 +39,8 @@ func NewNameSystem(r routing.IpfsRouting, ds ds.Datastore) NameSystem { } } +const DefaultResolverCacheTTL = time.Minute + // Resolve implements Resolver. func (ns *mpns) Resolve(ctx context.Context, name string) (path.Path, error) { return ns.ResolveN(ctx, name, DefaultDepthLimit) diff --git a/namesys/pb/namesys.pb.go b/namesys/pb/namesys.pb.go index 4d99a5e0a..0508e772d 100644 --- a/namesys/pb/namesys.pb.go +++ b/namesys/pb/namesys.pb.go @@ -57,6 +57,7 @@ type IpnsEntry struct { ValidityType *IpnsEntry_ValidityType `protobuf:"varint,3,opt,name=validityType,enum=namesys.pb.IpnsEntry_ValidityType" json:"validityType,omitempty"` Validity []byte `protobuf:"bytes,4,opt,name=validity" json:"validity,omitempty"` Sequence *uint64 `protobuf:"varint,5,opt,name=sequence" json:"sequence,omitempty"` + Ttl *uint64 `protobuf:"varint,6,opt,name=ttl" json:"ttl,omitempty"` XXX_unrecognized []byte `json:"-"` } @@ -99,6 +100,13 @@ func (m *IpnsEntry) GetSequence() uint64 { return 0 } +func (m *IpnsEntry) GetTtl() uint64 { + if m != nil && m.Ttl != nil { + return *m.Ttl + } + return 0 +} + func init() { proto.RegisterEnum("namesys.pb.IpnsEntry_ValidityType", IpnsEntry_ValidityType_name, IpnsEntry_ValidityType_value) } diff --git a/namesys/pb/namesys.proto b/namesys/pb/namesys.proto index 242f77bf2..d6eaf3243 100644 --- a/namesys/pb/namesys.proto +++ b/namesys/pb/namesys.proto @@ -12,4 +12,6 @@ message IpnsEntry { optional bytes validity = 4; optional uint64 sequence = 5; + + optional uint64 ttl = 6; } diff --git a/namesys/publisher.go b/namesys/publisher.go index e31217dff..78d7bb37c 100644 --- a/namesys/publisher.go +++ b/namesys/publisher.go @@ -121,6 +121,19 @@ func (p *ipnsPublisher) getPreviousSeqNo(ctx context.Context, ipnskey key.Key) ( return e.GetSequence(), nil } +// setting the TTL on published records is an experimental feature. +// as such, i'm using the context to wire it through to avoid changing too +// much code along the way. +func checkCtxTTL(ctx context.Context) (time.Duration, bool) { + v := ctx.Value("ipns-publish-ttl") + if v == nil { + return 0, false + } + + d, ok := v.(time.Duration) + return d, ok +} + func PutRecordToRouting(ctx context.Context, k ci.PrivKey, value path.Path, seqnum uint64, eol time.Time, r routing.IpfsRouting, id peer.ID) error { namekey, ipnskey := IpnsKeysForID(id) entry, err := CreateRoutingEntryData(k, value, seqnum, eol) @@ -128,6 +141,11 @@ func PutRecordToRouting(ctx context.Context, k ci.PrivKey, value path.Path, seqn return err } + ttl, ok := checkCtxTTL(ctx) + if ok { + entry.Ttl = proto.Uint64(uint64(ttl.Nanoseconds())) + } + err = PublishEntry(ctx, r, ipnskey, entry) if err != nil { return err diff --git a/namesys/republisher/repub_test.go b/namesys/republisher/repub_test.go index ef7fcf7e3..92c224a73 100644 --- a/namesys/republisher/repub_test.go +++ b/namesys/republisher/repub_test.go @@ -18,6 +18,8 @@ import ( ) func TestRepublish(t *testing.T) { + // set cache life to zero for testing low-period repubs + ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -34,6 +36,8 @@ func TestRepublish(t *testing.T) { t.Fatal(err) } + nd.Namesys = namesys.NewNameSystem(nd.Routing, nd.Repo.Datastore(), 0) + nodes = append(nodes, nd) } diff --git a/namesys/resolve_test.go b/namesys/resolve_test.go index 4d81751f1..11145ff01 100644 --- a/namesys/resolve_test.go +++ b/namesys/resolve_test.go @@ -19,7 +19,7 @@ func TestRoutingResolve(t *testing.T) { d := mockrouting.NewServer().Client(testutil.RandIdentityOrFatal(t)) dstore := ds.NewMapDatastore() - resolver := NewRoutingResolver(d) + resolver := NewRoutingResolver(d, 0) publisher := NewRoutingPublisher(d, dstore) privk, pubk, err := testutil.RandTestKeyPair(512) @@ -53,7 +53,7 @@ func TestPrexistingExpiredRecord(t *testing.T) { dstore := ds.NewMapDatastore() d := mockrouting.NewServer().ClientWithDatastore(context.Background(), testutil.RandIdentityOrFatal(t), dstore) - resolver := NewRoutingResolver(d) + resolver := NewRoutingResolver(d, 0) publisher := NewRoutingPublisher(d, dstore) privk, pubk, err := testutil.RandTestKeyPair(512) @@ -90,7 +90,7 @@ func TestPrexistingRecord(t *testing.T) { dstore := ds.NewMapDatastore() d := mockrouting.NewServer().ClientWithDatastore(context.Background(), testutil.RandIdentityOrFatal(t), dstore) - resolver := NewRoutingResolver(d) + resolver := NewRoutingResolver(d, 0) publisher := NewRoutingPublisher(d, dstore) privk, pubk, err := testutil.RandTestKeyPair(512) diff --git a/namesys/routing.go b/namesys/routing.go index 63a323cae..a25fa19f6 100644 --- a/namesys/routing.go +++ b/namesys/routing.go @@ -2,16 +2,19 @@ package namesys import ( "fmt" + "time" proto "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/gogo/protobuf/proto" + lru "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/hashicorp/golang-lru" mh "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multihash" "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context" + logging "github.com/ipfs/go-ipfs/vendor/QmTBXYb6y2ZcJmoXVKk3pf9rzSEjbCg7tQaJW7RSuH14nv/go-log" key "github.com/ipfs/go-ipfs/blocks/key" pb "github.com/ipfs/go-ipfs/namesys/pb" path "github.com/ipfs/go-ipfs/path" routing "github.com/ipfs/go-ipfs/routing" - logging "github.com/ipfs/go-ipfs/vendor/QmTBXYb6y2ZcJmoXVKk3pf9rzSEjbCg7tQaJW7RSuH14nv/go-log" + u "github.com/ipfs/go-ipfs/util" ) var log = logging.Logger("namesys") @@ -19,25 +22,84 @@ var log = logging.Logger("namesys") // routingResolver implements NSResolver for the main IPFS SFS-like naming type routingResolver struct { routing routing.IpfsRouting + + cache *lru.Cache +} + +func (r *routingResolver) cacheGet(name string) (path.Path, bool) { + if r.cache == nil { + return "", false + } + + ientry, ok := r.cache.Get(name) + if !ok { + return "", false + } + + entry, ok := ientry.(cacheEntry) + if !ok { + // should never happen, purely for sanity + log.Panicf("unexpected type %T in cache for %q.", ientry, name) + } + + if time.Now().Before(entry.eol) { + return entry.val, true + } + + r.cache.Remove(name) + + return "", false +} + +func (r *routingResolver) cacheSet(name string, val path.Path, rec *pb.IpnsEntry) { + if r.cache == nil { + return + } + + // if completely unspecified, just use one minute + ttl := DefaultResolverCacheTTL + if rec.Ttl != nil { + recttl := time.Duration(rec.GetTtl()) + if recttl >= 0 { + ttl = recttl + } + } + + cacheTil := time.Now().Add(ttl) + eol, ok := checkEOL(rec) + if ok && eol.Before(cacheTil) { + cacheTil = eol + } + + r.cache.Add(name, cacheEntry{ + val: val, + eol: cacheTil, + }) +} + +type cacheEntry struct { + val path.Path + eol time.Time } // NewRoutingResolver constructs a name resolver using the IPFS Routing system // to implement SFS-like naming on top. -func NewRoutingResolver(route routing.IpfsRouting) Resolver { +// cachesize is the limit of the number of entries in the lru cache. Setting it +// to '0' will disable caching. +func NewRoutingResolver(route routing.IpfsRouting, cachesize int) *routingResolver { if route == nil { panic("attempt to create resolver with nil routing system") } - return &routingResolver{routing: route} -} - -// newRoutingResolver returns a resolver instead of a Resolver. -func newRoutingResolver(route routing.IpfsRouting) resolver { - if route == nil { - panic("attempt to create resolver with nil routing system") + var cache *lru.Cache + if cachesize > 0 { + cache, _ = lru.New(cachesize) } - return &routingResolver{routing: route} + return &routingResolver{ + routing: route, + cache: cache, + } } // Resolve implements Resolver. @@ -54,6 +116,11 @@ func (r *routingResolver) ResolveN(ctx context.Context, name string, depth int) // resolve SFS-like names. func (r *routingResolver) resolveOnce(ctx context.Context, name string) (path.Path, error) { log.Debugf("RoutingResolve: '%s'", name) + cached, ok := r.cacheGet(name) + if ok { + return cached, nil + } + hash, err := mh.FromB58String(name) if err != nil { log.Warning("RoutingResolve: bad input hash: [%s]\n", name) @@ -98,10 +165,29 @@ func (r *routingResolver) resolveOnce(ctx context.Context, name string) (path.Pa valh, err := mh.Cast(entry.GetValue()) if err != nil { // Not a multihash, probably a new record - return path.ParsePath(string(entry.GetValue())) + p, err := path.ParsePath(string(entry.GetValue())) + if err != nil { + return "", err + } + + r.cacheSet(name, p, entry) + return p, nil } else { // Its an old style multihash record log.Warning("Detected old style multihash record") - return path.FromKey(key.Key(valh)), nil + p := path.FromKey(key.Key(valh)) + r.cacheSet(name, p, entry) + return p, nil } } + +func checkEOL(e *pb.IpnsEntry) (time.Time, bool) { + if e.GetValidityType() == pb.IpnsEntry_EOL { + eol, err := u.ParseRFC3339(string(e.GetValidity())) + if err != nil { + return time.Time{}, false + } + return eol, true + } + return time.Time{}, false +} diff --git a/repo/config/init.go b/repo/config/init.go index 49d0a1ecb..970597a65 100644 --- a/repo/config/init.go +++ b/repo/config/init.go @@ -64,6 +64,10 @@ func Init(out io.Writer, nBitsForKeypair int) (*Config, error) { IPNS: "/ipns", }, + Ipns: Ipns{ + ResolveCacheSize: 128, + }, + // tracking ipfs version used to generate the init folder and adding // update checker default setting. Version: VersionDefaultValue(), diff --git a/repo/config/ipns.go b/repo/config/ipns.go index feab6f044..44a95b099 100644 --- a/repo/config/ipns.go +++ b/repo/config/ipns.go @@ -3,4 +3,6 @@ package config type Ipns struct { RepublishPeriod string RecordLifetime string + + ResolveCacheSize int } diff --git a/test/sharness/t0240-republisher.sh b/test/sharness/t0240-republisher.sh index bb643d17e..c1db0fbd3 100755 --- a/test/sharness/t0240-republisher.sh +++ b/test/sharness/t0240-republisher.sh @@ -26,6 +26,7 @@ setup_iptb() { for i in $(test_seq 0 3) do ipfsi $i config Ipns.RepublishPeriod 20s + ipfsi $i config --json Ipns.ResolveCacheSize 0 done '