package coreunix import ( "context" "fmt" "io" "io/ioutil" "os" gopath "path" "path/filepath" "strconv" core "github.com/ipfs/go-ipfs/core" coreiface "github.com/ipfs/go-ipfs/core/coreapi/interface" "github.com/ipfs/go-ipfs/pin" chunker "gx/ipfs/QmR4QQVkBZsZENRjYFVi8dEtPL3daZRNKk24m4r6WKJHNm/go-ipfs-chunker" posinfo "gx/ipfs/QmR6YMs8EkXQLXNwQKxLnQp2VBZSepoEJ8KCZAyanJHhJu/go-ipfs-posinfo" cid "gx/ipfs/QmR8BauakNcBa3RbE4nbQu76PDiJgoQgz8AJdhJuiU4TAw/go-cid" bstore "gx/ipfs/QmSNLNnL3kq3A1NGdQA9AtgxM9CWKiiSEup3W435jCkRQS/go-ipfs-blockstore" unixfs "gx/ipfs/QmXLCwhHh7bxRsBnCKNE9BAN87V44aSxXLquZYTtjr6fZ3/go-unixfs" balanced "gx/ipfs/QmXLCwhHh7bxRsBnCKNE9BAN87V44aSxXLquZYTtjr6fZ3/go-unixfs/importer/balanced" ihelper "gx/ipfs/QmXLCwhHh7bxRsBnCKNE9BAN87V44aSxXLquZYTtjr6fZ3/go-unixfs/importer/helpers" trickle "gx/ipfs/QmXLCwhHh7bxRsBnCKNE9BAN87V44aSxXLquZYTtjr6fZ3/go-unixfs/importer/trickle" files "gx/ipfs/QmZMWMvWMVKCbHetJ4RgndbuEF1io2UpUxwQwtNjtYPzSC/go-ipfs-files" dag "gx/ipfs/QmaDBne4KeY3UepeqSVKYpSmQGa3q9zP6x3LfVF2UjF3Hc/go-merkledag" ipld "gx/ipfs/QmcKKBwfz6FyQdHR2jsXrrF6XeSBXYL86anmWNewpFpoF5/go-ipld-format" mfs "gx/ipfs/QmcUXFi2Fp7oguoFT81f2poJpnb44dFkZanQhDBHMoYyG9/go-mfs" logging "gx/ipfs/QmcuXC5cxs79ro2cUuHs4HQ2bkDLJUYokwL8aivcX6HW3C/go-log" ) var log = logging.Logger("coreunix") // how many bytes of progress to wait before sending a progress update message const progressReaderIncrement = 1024 * 256 var liveCacheSize = uint64(256 << 10) type Link struct { Name, Hash string Size uint64 } type Object struct { Hash string Links []Link Size string } // NewAdder Returns a new Adder used for a file add operation. func NewAdder(ctx context.Context, p pin.Pinner, bs bstore.GCBlockstore, ds ipld.DAGService) (*Adder, error) { bufferedDS := ipld.NewBufferedDAG(ctx, ds) return &Adder{ ctx: ctx, pinning: p, blockstore: bs, dagService: ds, bufferedDS: bufferedDS, Progress: false, Hidden: true, Pin: true, Trickle: false, Wrap: false, Chunker: "", }, nil } // Adder holds the switches passed to the `add` command. type Adder struct { ctx context.Context pinning pin.Pinner blockstore bstore.GCBlockstore dagService ipld.DAGService bufferedDS *ipld.BufferedDAG Out chan<- interface{} Progress bool Hidden bool Pin bool Trickle bool RawLeaves bool Silent bool Wrap bool Name string NoCopy bool Chunker string root ipld.Node mroot *mfs.Root unlocker bstore.Unlocker tempRoot cid.Cid CidBuilder cid.Builder liveNodes uint64 } func (adder *Adder) mfsRoot() (*mfs.Root, error) { if adder.mroot != nil { return adder.mroot, nil } rnode := unixfs.EmptyDirNode() rnode.SetCidBuilder(adder.CidBuilder) mr, err := mfs.NewRoot(adder.ctx, adder.dagService, rnode, nil) if err != nil { return nil, err } adder.mroot = mr return adder.mroot, nil } // SetMfsRoot sets `r` as the root for Adder. func (adder *Adder) SetMfsRoot(r *mfs.Root) { adder.mroot = r } // Constructs a node from reader's data, and adds it. Doesn't pin. func (adder *Adder) add(reader io.Reader) (ipld.Node, error) { chnk, err := chunker.FromString(reader, adder.Chunker) if err != nil { return nil, err } // Make sure all added nodes are written when done. defer adder.bufferedDS.Commit() params := ihelper.DagBuilderParams{ Dagserv: adder.bufferedDS, RawLeaves: adder.RawLeaves, Maxlinks: ihelper.DefaultLinksPerBlock, NoCopy: adder.NoCopy, CidBuilder: adder.CidBuilder, } if adder.Trickle { return trickle.Layout(params.New(chnk)) } return balanced.Layout(params.New(chnk)) } // RootNode returns the root node of the Added. func (adder *Adder) RootNode() (ipld.Node, error) { // for memoizing if adder.root != nil { return adder.root, nil } mr, err := adder.mfsRoot() if err != nil { return nil, err } root, err := mr.GetDirectory().GetNode() if err != nil { return nil, err } // if not wrapping, AND one root file, use that hash as root. if !adder.Wrap && len(root.Links()) == 1 { nd, err := root.Links()[0].GetNode(adder.ctx, adder.dagService) if err != nil { return nil, err } root = nd } adder.root = root return root, err } // Recursively pins the root node of Adder and // writes the pin state to the backing datastore. func (adder *Adder) PinRoot() error { root, err := adder.RootNode() if err != nil { return err } if !adder.Pin { return nil } rnk := root.Cid() err = adder.dagService.Add(adder.ctx, root) if err != nil { return err } if adder.tempRoot.Defined() { err := adder.pinning.Unpin(adder.ctx, adder.tempRoot, true) if err != nil { return err } adder.tempRoot = rnk } adder.pinning.PinWithMode(rnk, pin.Recursive) return adder.pinning.Flush() } // Finalize flushes the mfs root directory and returns the mfs root node. func (adder *Adder) Finalize() (ipld.Node, error) { mr, err := adder.mfsRoot() if err != nil { return nil, err } var root mfs.FSNode rootdir := mr.GetDirectory() root = rootdir err = root.Flush() if err != nil { return nil, err } var name string if !adder.Wrap { children, err := rootdir.ListNames(adder.ctx) if err != nil { return nil, err } if len(children) == 0 { return nil, fmt.Errorf("expected at least one child dir, got none") } // Replace root with the first child name = children[0] root, err = rootdir.Child(name) if err != nil { return nil, err } } err = adder.outputDirs(name, root) if err != nil { return nil, err } err = mr.Close() if err != nil { return nil, err } return root.GetNode() } func (adder *Adder) outputDirs(path string, fsn mfs.FSNode) error { switch fsn := fsn.(type) { case *mfs.File: return nil case *mfs.Directory: names, err := fsn.ListNames(adder.ctx) if err != nil { return err } for _, name := range names { child, err := fsn.Child(name) if err != nil { return err } childpath := gopath.Join(path, name) err = adder.outputDirs(childpath, child) if err != nil { return err } fsn.Uncache(name) } nd, err := fsn.GetNode() if err != nil { return err } return outputDagnode(adder.Out, path, nd) default: return fmt.Errorf("unrecognized fsn type: %#v", fsn) } } // Add builds a merkledag node from a reader, adds it to the blockstore, // and returns the key representing that node. // If you want to pin it, use NewAdder() and Adder.PinRoot(). func Add(n *core.IpfsNode, r io.Reader) (string, error) { return AddWithContext(n.Context(), n, r) } // AddWithContext does the same as Add, but with a custom context. func AddWithContext(ctx context.Context, n *core.IpfsNode, r io.Reader) (string, error) { defer n.Blockstore.PinLock().Unlock() fileAdder, err := NewAdder(ctx, n.Pinning, n.Blockstore, n.DAG) if err != nil { return "", err } node, err := fileAdder.add(r) if err != nil { return "", err } return node.Cid().String(), nil } // AddR recursively adds files in |path|. func AddR(n *core.IpfsNode, root string) (key string, err error) { defer n.Blockstore.PinLock().Unlock() stat, err := os.Lstat(root) if err != nil { return "", err } f, err := files.NewSerialFile(filepath.Base(root), root, false, stat) if err != nil { return "", err } defer f.Close() fileAdder, err := NewAdder(n.Context(), n.Pinning, n.Blockstore, n.DAG) if err != nil { return "", err } err = fileAdder.addFile(f) if err != nil { return "", err } nd, err := fileAdder.Finalize() if err != nil { return "", err } return nd.String(), nil } // AddWrapped adds data from a reader, and wraps it with a directory object // to preserve the filename. // Returns the path of the added file ("