From c8cfed5c84f1d4c991434184cc9e287d9bdebeff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 10 Jan 2018 18:41:06 +0100 Subject: [PATCH] coreapi: implement pin api MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- core/coreapi/coreapi.go | 4 + core/coreapi/interface/interface.go | 25 +++- core/coreapi/interface/options/pin.go | 27 ++++ core/coreapi/pin.go | 179 ++++++++++++++++++++++++-- 4 files changed, 221 insertions(+), 14 deletions(-) diff --git a/core/coreapi/coreapi.go b/core/coreapi/coreapi.go index cc0ab39ce..bccb330cd 100644 --- a/core/coreapi/coreapi.go +++ b/core/coreapi/coreapi.go @@ -52,6 +52,10 @@ func (api *CoreAPI) Object() coreiface.ObjectAPI { return &ObjectAPI{api, nil} } +func (api *CoreAPI) Pin() coreiface.PinAPI { + return &PinAPI{api, nil} +} + // ResolveNode resolves the path `p` using Unixfx resolver, gets and returns the // resolved Node. func (api *CoreAPI) ResolveNode(ctx context.Context, p coreiface.Path) (coreiface.Node, error) { diff --git a/core/coreapi/interface/interface.go b/core/coreapi/interface/interface.go index 40fa4131e..75a168bf3 100644 --- a/core/coreapi/interface/interface.go +++ b/core/coreapi/interface/interface.go @@ -67,6 +67,24 @@ type Pin interface { Type() string } +// PinStatus holds information about pin health +type PinStatus interface { + // Ok indicates whether the pin has been verified to be correct + Ok() bool + + // BadNodes returns any bad (usually missing) nodes from the pin + BadNodes() []BadPinNode +} + +// BadPinNode is a node that has been marked as bad by Pin.Verify +type BadPinNode interface { + // Path is the path of the node + Path() Path + + // Err is the reason why the node has been marked as bad + Err() error +} + // CoreAPI defines an unified interface to IPFS for Go programs. type CoreAPI interface { // Unixfs returns an implementation of Unixfs API. @@ -83,6 +101,7 @@ type CoreAPI interface { // Key returns an implementation of Key API. Key() KeyAPI + Pin() PinAPI // ObjectAPI returns an implementation of Object API Object() ObjectAPI @@ -342,7 +361,7 @@ type PinAPI interface { WithRecursive(bool) options.PinAddOption // Ls returns list of pinned objects on this node - Ls(context.Context) ([]Pin, error) + Ls(context.Context, ...options.PinLsOption) ([]Pin, error) // WithType is an option for Ls which allows to specify which pin types should // be returned @@ -360,10 +379,10 @@ type PinAPI interface { // Update changes one pin to another, skipping checks for matching paths in // the old tree - Update(ctx context.Context, from Path, to Path) error + Update(ctx context.Context, from Path, to Path, opts ...options.PinUpdateOption) error // Verify verifies the integrity of pinned objects - Verify(context.Context) error + Verify(context.Context) (<-chan PinStatus, error) } var ErrIsDir = errors.New("object is a directory") diff --git a/core/coreapi/interface/options/pin.go b/core/coreapi/interface/options/pin.go index 4ad16d555..f97f7b16e 100644 --- a/core/coreapi/interface/options/pin.go +++ b/core/coreapi/interface/options/pin.go @@ -8,8 +8,13 @@ type PinLsSettings struct { Type string } +type PinUpdateSettings struct { + Unpin bool +} + type PinAddOption func(*PinAddSettings) error type PinLsOption func(settings *PinLsSettings) error +type PinUpdateOption func(*PinUpdateSettings) error func PinAddOptions(opts ...PinAddOption) (*PinAddSettings, error) { options := &PinAddSettings{ @@ -41,6 +46,21 @@ func PinLsOptions(opts ...PinLsOption) (*PinLsSettings, error) { return options, nil } +func PinUpdateOptions(opts ...PinUpdateOption) (*PinUpdateSettings, error) { + options := &PinUpdateSettings{ + Unpin: true, + } + + for _, opt := range opts { + err := opt(options) + if err != nil { + return nil, err + } + } + + return options, nil +} + type PinOptions struct{} func (api *PinOptions) WithRecursive(recucsive bool) PinAddOption { @@ -56,3 +76,10 @@ func (api *PinOptions) WithType(t string) PinLsOption { return nil } } + +func (api *PinOptions) WithUnpin(unpin bool) PinUpdateOption { + return func(settings *PinUpdateSettings) error { + settings.Unpin = unpin + return nil + } +} diff --git a/core/coreapi/pin.go b/core/coreapi/pin.go index ef7f98c24..bcfba1ff8 100644 --- a/core/coreapi/pin.go +++ b/core/coreapi/pin.go @@ -2,10 +2,15 @@ package coreapi import ( "context" + "fmt" coreiface "github.com/ipfs/go-ipfs/core/coreapi/interface" caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options" - "github.com/pkg/errors" + corerepo "github.com/ipfs/go-ipfs/core/corerepo" + merkledag "github.com/ipfs/go-ipfs/merkledag" + pin "github.com/ipfs/go-ipfs/pin" + + cid "gx/ipfs/QmeSrf6pzut73u6zLQkRFQ3ygt3k6XFT2kjdYP8Tnkwwyg/go-cid" ) type PinAPI struct { @@ -13,22 +18,174 @@ type PinAPI struct { *caopts.PinOptions } -func (api *PinAPI) Add(context.Context, coreiface.Path, ...caopts.PinAddOption) error { - return errors.New("TODO") +func (api *PinAPI) Add(ctx context.Context, p coreiface.Path, opts ...caopts.PinAddOption) error { + settings, err := caopts.PinAddOptions(opts...) + if err != nil { + return err + } + + defer api.node.Blockstore.PinLock().Unlock() + + _, err = corerepo.Pin(api.node, ctx, []string{p.String()}, settings.Recursive) + if err != nil { + return err + } + + return nil } -func (api *PinAPI) Ls(context.Context) ([]coreiface.Pin, error) { - return nil, errors.New("TODO") +func (api *PinAPI) Ls(ctx context.Context, opts ...caopts.PinLsOption) ([]coreiface.Pin, error) { + settings, err := caopts.PinLsOptions(opts...) + if err != nil { + return nil, err + } + + switch settings.Type { + case "all", "direct", "indirect", "recursive": + default: + return nil, fmt.Errorf("invalid type '%s', must be one of {direct, indirect, recursive, all}", settings.Type) + } + + return pinLsAll(settings.Type, ctx, api.node.Pinning, api.node.DAG) } -func (api *PinAPI) Rm(context.Context, coreiface.Path) error { - return errors.New("TODO") +func (api *PinAPI) Rm(ctx context.Context, p coreiface.Path) error { + _, err := corerepo.Unpin(api.node, ctx, []string{p.String()}, true) + if err != nil { + return err + } + + return nil } -func (api *PinAPI) Update(ctx context.Context, from coreiface.Path, to coreiface.Path) error { - return errors.New("TODO") +func (api *PinAPI) Update(ctx context.Context, from coreiface.Path, to coreiface.Path, opts ...caopts.PinUpdateOption) error { + settings, err := caopts.PinUpdateOptions(opts...) + if err != nil { + return err + } + + return api.node.Pinning.Update(ctx, from.Cid(), to.Cid(), settings.Unpin) } -func (api *PinAPI) Verify(context.Context) error { - return errors.New("TODO") +type pinStatus struct { + cid *cid.Cid + ok bool + badNodes []coreiface.BadPinNode +} + +// BadNode is used in PinVerifyRes +type badNode struct { + cid *cid.Cid + err error +} + +func (s *pinStatus) Ok() bool { + return s.Ok() +} + +func (s *pinStatus) BadNodes() []coreiface.BadPinNode { + return s.badNodes +} + +func (n *badNode) Path() coreiface.Path { + return ParseCid(n.cid) +} + +func (n *badNode) Err() error { + return n.err +} + +func (api *PinAPI) Verify(ctx context.Context) (<-chan coreiface.PinStatus, error) { + visited := make(map[string]*pinStatus) + getLinks := api.node.DAG.GetOfflineLinkService().GetLinks + recPins := api.node.Pinning.RecursiveKeys() + + var checkPin func(root *cid.Cid) *pinStatus + checkPin = func(root *cid.Cid) *pinStatus { + key := root.String() + if status, ok := visited[key]; ok { + return status + } + + links, err := getLinks(ctx, root) + if err != nil { + status := &pinStatus{ok: false, cid: root} + status.badNodes = []coreiface.BadPinNode{&badNode{cid: root, err: err}} + visited[key] = status + return status + } + + status := &pinStatus{ok: true, cid: root} + for _, lnk := range links { + res := checkPin(lnk.Cid) + if !res.ok { + status.ok = false + status.badNodes = append(status.badNodes, res.badNodes...) + } + } + + visited[key] = status + return status + } + + out := make(chan coreiface.PinStatus) + go func() { + defer close(out) + for _, c := range recPins { + out <- checkPin(c) + } + }() + + return out, nil +} + +type pinInfo struct { + pinType string + object *cid.Cid +} + +func (p *pinInfo) Path() coreiface.Path { + return ParseCid(p.object) +} + +func (p *pinInfo) Type() string { + return p.pinType +} + +func pinLsAll(typeStr string, ctx context.Context, pinning pin.Pinner, dag merkledag.DAGService) ([]coreiface.Pin, error) { + + keys := make(map[string]*pinInfo) + + AddToResultKeys := func(keyList []*cid.Cid, typeStr string) { + for _, c := range keyList { + keys[c.String()] = &pinInfo{ + pinType: typeStr, + object: c, + } + } + } + + if typeStr == "direct" || typeStr == "all" { + AddToResultKeys(pinning.DirectKeys(), "direct") + } + if typeStr == "indirect" || typeStr == "all" { + set := cid.NewSet() + for _, k := range pinning.RecursiveKeys() { + err := merkledag.EnumerateChildren(ctx, dag.GetLinks, k, set.Visit) + if err != nil { + return nil, err + } + } + AddToResultKeys(set.Keys(), "indirect") + } + if typeStr == "recursive" || typeStr == "all" { + AddToResultKeys(pinning.RecursiveKeys(), "recursive") + } + + out := make([]coreiface.Pin, 0, len(keys)) + for _, v := range keys { + out = append(out, v) + } + + return out, nil }