mirror of
https://github.com/ipfs/kubo.git
synced 2025-05-21 17:08:13 +08:00
428 lines
12 KiB
Go
428 lines
12 KiB
Go
package commands
|
|
|
|
import (
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"path"
|
|
"strings"
|
|
|
|
"github.com/ipfs/go-ipfs/core/commands/cmdenv"
|
|
|
|
cmds "github.com/ipfs/go-ipfs-cmds"
|
|
files "github.com/ipfs/go-ipfs-files"
|
|
coreiface "github.com/ipfs/interface-go-ipfs-core"
|
|
"github.com/ipfs/interface-go-ipfs-core/options"
|
|
mh "github.com/multiformats/go-multihash"
|
|
pb "gopkg.in/cheggaaa/pb.v1"
|
|
)
|
|
|
|
// ErrDepthLimitExceeded indicates that the max depth has been exceeded.
|
|
var ErrDepthLimitExceeded = fmt.Errorf("depth limit exceeded")
|
|
|
|
type AddEvent struct {
|
|
Name string
|
|
Hash string `json:",omitempty"`
|
|
Bytes int64 `json:",omitempty"`
|
|
Size string `json:",omitempty"`
|
|
}
|
|
|
|
const (
|
|
quietOptionName = "quiet"
|
|
quieterOptionName = "quieter"
|
|
silentOptionName = "silent"
|
|
progressOptionName = "progress"
|
|
trickleOptionName = "trickle"
|
|
wrapOptionName = "wrap-with-directory"
|
|
onlyHashOptionName = "only-hash"
|
|
chunkerOptionName = "chunker"
|
|
pinOptionName = "pin"
|
|
rawLeavesOptionName = "raw-leaves"
|
|
noCopyOptionName = "nocopy"
|
|
fstoreCacheOptionName = "fscache"
|
|
cidVersionOptionName = "cid-version"
|
|
hashOptionName = "hash"
|
|
inlineOptionName = "inline"
|
|
inlineLimitOptionName = "inline-limit"
|
|
)
|
|
|
|
const adderOutChanSize = 8
|
|
|
|
var AddCmd = &cmds.Command{
|
|
Helptext: cmds.HelpText{
|
|
Tagline: "Add a file or directory to ipfs.",
|
|
ShortDescription: `
|
|
Adds contents of <path> to ipfs. Use -r to add directories (recursively).
|
|
`,
|
|
LongDescription: `
|
|
Adds contents of <path> to ipfs. Use -r to add directories.
|
|
Note that directories are added recursively, to form the ipfs
|
|
MerkleDAG.
|
|
|
|
The wrap option, '-w', wraps the file (or files, if using the
|
|
recursive option) in a directory. This directory contains only
|
|
the files which have been added, and means that the file retains
|
|
its filename. For example:
|
|
|
|
> ipfs add example.jpg
|
|
added QmbFMke1KXqnYyBBWxB74N4c5SBnJMVAiMNRcGu6x1AwQH example.jpg
|
|
> ipfs add example.jpg -w
|
|
added QmbFMke1KXqnYyBBWxB74N4c5SBnJMVAiMNRcGu6x1AwQH example.jpg
|
|
added QmaG4FuMqEBnQNn3C8XJ5bpW8kLs7zq2ZXgHptJHbKDDVx
|
|
|
|
You can now refer to the added file in a gateway, like so:
|
|
|
|
/ipfs/QmaG4FuMqEBnQNn3C8XJ5bpW8kLs7zq2ZXgHptJHbKDDVx/example.jpg
|
|
|
|
The chunker option, '-s', specifies the chunking strategy that dictates
|
|
how to break files into blocks. Blocks with same content can
|
|
be deduplicated. Different chunking strategies will produce different
|
|
hashes for the same file. The default is a fixed block size of
|
|
256 * 1024 bytes, 'size-262144'. Alternatively, you can use the
|
|
Buzhash or Rabin fingerprint chunker for content defined chunking by
|
|
specifying buzhash or rabin-[min]-[avg]-[max] (where min/avg/max refer
|
|
to the desired chunk sizes in bytes), e.g. 'rabin-262144-524288-1048576'.
|
|
|
|
The following examples use very small byte sizes to demonstrate the
|
|
properties of the different chunkers on a small file. You'll likely
|
|
want to use a 1024 times larger chunk sizes for most files.
|
|
|
|
> ipfs add --chunker=size-2048 ipfs-logo.svg
|
|
added QmafrLBfzRLV4XSH1XcaMMeaXEUhDJjmtDfsYU95TrWG87 ipfs-logo.svg
|
|
> ipfs add --chunker=rabin-512-1024-2048 ipfs-logo.svg
|
|
added Qmf1hDN65tR55Ubh2RN1FPxr69xq3giVBz1KApsresY8Gn ipfs-logo.svg
|
|
|
|
You can now check what blocks have been created by:
|
|
|
|
> ipfs object links QmafrLBfzRLV4XSH1XcaMMeaXEUhDJjmtDfsYU95TrWG87
|
|
QmY6yj1GsermExDXoosVE3aSPxdMNYr6aKuw3nA8LoWPRS 2059
|
|
Qmf7ZQeSxq2fJVJbCmgTrLLVN9tDR9Wy5k75DxQKuz5Gyt 1195
|
|
> ipfs object links Qmf1hDN65tR55Ubh2RN1FPxr69xq3giVBz1KApsresY8Gn
|
|
QmY6yj1GsermExDXoosVE3aSPxdMNYr6aKuw3nA8LoWPRS 2059
|
|
QmerURi9k4XzKCaaPbsK6BL5pMEjF7PGphjDvkkjDtsVf3 868
|
|
QmQB28iwSriSUSMqG2nXDTLtdPHgWb4rebBrU7Q1j4vxPv 338
|
|
|
|
Finally, a note on hash determinism. While not guaranteed, adding the same
|
|
file/directory with the same flags will almost always result in the same output
|
|
hash. However, almost all of the flags provided by this command (other than pin,
|
|
only-hash, and progress/status related flags) will change the final hash.
|
|
`,
|
|
},
|
|
|
|
Arguments: []cmds.Argument{
|
|
cmds.FileArg("path", true, true, "The path to a file to be added to ipfs.").EnableRecursive().EnableStdin(),
|
|
},
|
|
Options: []cmds.Option{
|
|
cmds.OptionRecursivePath, // a builtin option that allows recursive paths (-r, --recursive)
|
|
cmds.OptionDerefArgs, // a builtin option that resolves passed in filesystem links (--dereference-args)
|
|
cmds.OptionStdinName, // a builtin option that optionally allows wrapping stdin into a named file
|
|
cmds.OptionHidden,
|
|
cmds.OptionIgnore,
|
|
cmds.OptionIgnoreRules,
|
|
cmds.BoolOption(quietOptionName, "q", "Write minimal output."),
|
|
cmds.BoolOption(quieterOptionName, "Q", "Write only final hash."),
|
|
cmds.BoolOption(silentOptionName, "Write no output."),
|
|
cmds.BoolOption(progressOptionName, "p", "Stream progress data."),
|
|
cmds.BoolOption(trickleOptionName, "t", "Use trickle-dag format for dag generation."),
|
|
cmds.BoolOption(onlyHashOptionName, "n", "Only chunk and hash - do not write to disk."),
|
|
cmds.BoolOption(wrapOptionName, "w", "Wrap files with a directory object."),
|
|
cmds.StringOption(chunkerOptionName, "s", "Chunking algorithm, size-[bytes], rabin-[min]-[avg]-[max] or buzhash").WithDefault("size-262144"),
|
|
cmds.BoolOption(pinOptionName, "Pin this object when adding.").WithDefault(true),
|
|
cmds.BoolOption(rawLeavesOptionName, "Use raw blocks for leaf nodes. (experimental)"),
|
|
cmds.BoolOption(noCopyOptionName, "Add the file using filestore. Implies raw-leaves. (experimental)"),
|
|
cmds.BoolOption(fstoreCacheOptionName, "Check the filestore for pre-existing blocks. (experimental)"),
|
|
cmds.IntOption(cidVersionOptionName, "CID version. Defaults to 0 unless an option that depends on CIDv1 is passed. (experimental)"),
|
|
cmds.StringOption(hashOptionName, "Hash function to use. Implies CIDv1 if not sha2-256. (experimental)").WithDefault("sha2-256"),
|
|
cmds.BoolOption(inlineOptionName, "Inline small blocks into CIDs. (experimental)"),
|
|
cmds.IntOption(inlineLimitOptionName, "Maximum block size to inline. (experimental)").WithDefault(32),
|
|
},
|
|
PreRun: func(req *cmds.Request, env cmds.Environment) error {
|
|
quiet, _ := req.Options[quietOptionName].(bool)
|
|
quieter, _ := req.Options[quieterOptionName].(bool)
|
|
quiet = quiet || quieter
|
|
|
|
silent, _ := req.Options[silentOptionName].(bool)
|
|
|
|
if quiet || silent {
|
|
return nil
|
|
}
|
|
|
|
// ipfs cli progress bar defaults to true unless quiet or silent is used
|
|
_, found := req.Options[progressOptionName].(bool)
|
|
if !found {
|
|
req.Options[progressOptionName] = true
|
|
}
|
|
|
|
return nil
|
|
},
|
|
Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
|
|
api, err := cmdenv.GetApi(env, req)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
progress, _ := req.Options[progressOptionName].(bool)
|
|
trickle, _ := req.Options[trickleOptionName].(bool)
|
|
wrap, _ := req.Options[wrapOptionName].(bool)
|
|
hash, _ := req.Options[onlyHashOptionName].(bool)
|
|
silent, _ := req.Options[silentOptionName].(bool)
|
|
chunker, _ := req.Options[chunkerOptionName].(string)
|
|
dopin, _ := req.Options[pinOptionName].(bool)
|
|
rawblks, rbset := req.Options[rawLeavesOptionName].(bool)
|
|
nocopy, _ := req.Options[noCopyOptionName].(bool)
|
|
fscache, _ := req.Options[fstoreCacheOptionName].(bool)
|
|
cidVer, cidVerSet := req.Options[cidVersionOptionName].(int)
|
|
hashFunStr, _ := req.Options[hashOptionName].(string)
|
|
inline, _ := req.Options[inlineOptionName].(bool)
|
|
inlineLimit, _ := req.Options[inlineLimitOptionName].(int)
|
|
|
|
hashFunCode, ok := mh.Names[strings.ToLower(hashFunStr)]
|
|
if !ok {
|
|
return fmt.Errorf("unrecognized hash function: %s", strings.ToLower(hashFunStr))
|
|
}
|
|
|
|
enc, err := cmdenv.GetCidEncoder(req)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
toadd := req.Files
|
|
if wrap {
|
|
toadd = files.NewSliceDirectory([]files.DirEntry{
|
|
files.FileEntry("", req.Files),
|
|
})
|
|
}
|
|
|
|
opts := []options.UnixfsAddOption{
|
|
options.Unixfs.Hash(hashFunCode),
|
|
|
|
options.Unixfs.Inline(inline),
|
|
options.Unixfs.InlineLimit(inlineLimit),
|
|
|
|
options.Unixfs.Chunker(chunker),
|
|
|
|
options.Unixfs.Pin(dopin),
|
|
options.Unixfs.HashOnly(hash),
|
|
options.Unixfs.FsCache(fscache),
|
|
options.Unixfs.Nocopy(nocopy),
|
|
|
|
options.Unixfs.Progress(progress),
|
|
options.Unixfs.Silent(silent),
|
|
}
|
|
|
|
if cidVerSet {
|
|
opts = append(opts, options.Unixfs.CidVersion(cidVer))
|
|
}
|
|
|
|
if rbset {
|
|
opts = append(opts, options.Unixfs.RawLeaves(rawblks))
|
|
}
|
|
|
|
if trickle {
|
|
opts = append(opts, options.Unixfs.Layout(options.TrickleLayout))
|
|
}
|
|
|
|
opts = append(opts, nil) // events option placeholder
|
|
|
|
var added int
|
|
addit := toadd.Entries()
|
|
for addit.Next() {
|
|
_, dir := addit.Node().(files.Directory)
|
|
errCh := make(chan error, 1)
|
|
events := make(chan interface{}, adderOutChanSize)
|
|
opts[len(opts)-1] = options.Unixfs.Events(events)
|
|
|
|
go func() {
|
|
var err error
|
|
defer close(events)
|
|
_, err = api.Unixfs().Add(req.Context, addit.Node(), opts...)
|
|
errCh <- err
|
|
}()
|
|
|
|
for event := range events {
|
|
output, ok := event.(*coreiface.AddEvent)
|
|
if !ok {
|
|
return errors.New("unknown event type")
|
|
}
|
|
|
|
h := ""
|
|
if output.Path != nil {
|
|
h = enc.Encode(output.Path.Cid())
|
|
}
|
|
|
|
if !dir && addit.Name() != "" {
|
|
output.Name = addit.Name()
|
|
} else {
|
|
output.Name = path.Join(addit.Name(), output.Name)
|
|
}
|
|
|
|
if err := res.Emit(&AddEvent{
|
|
Name: output.Name,
|
|
Hash: h,
|
|
Bytes: output.Bytes,
|
|
Size: output.Size,
|
|
}); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
if err := <-errCh; err != nil {
|
|
return err
|
|
}
|
|
added++
|
|
}
|
|
|
|
if addit.Err() != nil {
|
|
return addit.Err()
|
|
}
|
|
|
|
if added == 0 {
|
|
return fmt.Errorf("expected a file argument")
|
|
}
|
|
|
|
return nil
|
|
},
|
|
PostRun: cmds.PostRunMap{
|
|
cmds.CLI: func(res cmds.Response, re cmds.ResponseEmitter) error {
|
|
sizeChan := make(chan int64, 1)
|
|
outChan := make(chan interface{})
|
|
req := res.Request()
|
|
|
|
// Could be slow.
|
|
go func() {
|
|
size, err := req.Files.Size()
|
|
if err != nil {
|
|
log.Warnf("error getting files size: %s", err)
|
|
// see comment above
|
|
return
|
|
}
|
|
|
|
sizeChan <- size
|
|
}()
|
|
|
|
progressBar := func(wait chan struct{}) {
|
|
defer close(wait)
|
|
|
|
quiet, _ := req.Options[quietOptionName].(bool)
|
|
quieter, _ := req.Options[quieterOptionName].(bool)
|
|
quiet = quiet || quieter
|
|
|
|
progress, _ := req.Options[progressOptionName].(bool)
|
|
|
|
var bar *pb.ProgressBar
|
|
if progress {
|
|
bar = pb.New64(0).SetUnits(pb.U_BYTES)
|
|
bar.ManualUpdate = true
|
|
bar.ShowTimeLeft = false
|
|
bar.ShowPercent = false
|
|
bar.Output = os.Stderr
|
|
bar.Start()
|
|
}
|
|
|
|
lastFile := ""
|
|
lastHash := ""
|
|
var totalProgress, prevFiles, lastBytes int64
|
|
|
|
LOOP:
|
|
for {
|
|
select {
|
|
case out, ok := <-outChan:
|
|
if !ok {
|
|
if quieter {
|
|
fmt.Fprintln(os.Stdout, lastHash)
|
|
}
|
|
|
|
break LOOP
|
|
}
|
|
output := out.(*AddEvent)
|
|
if len(output.Hash) > 0 {
|
|
lastHash = output.Hash
|
|
if quieter {
|
|
continue
|
|
}
|
|
|
|
if progress {
|
|
// clear progress bar line before we print "added x" output
|
|
fmt.Fprintf(os.Stderr, "\033[2K\r")
|
|
}
|
|
if quiet {
|
|
fmt.Fprintf(os.Stdout, "%s\n", output.Hash)
|
|
} else {
|
|
fmt.Fprintf(os.Stdout, "added %s %s\n", output.Hash, output.Name)
|
|
}
|
|
|
|
} else {
|
|
if !progress {
|
|
continue
|
|
}
|
|
|
|
if len(lastFile) == 0 {
|
|
lastFile = output.Name
|
|
}
|
|
if output.Name != lastFile || output.Bytes < lastBytes {
|
|
prevFiles += lastBytes
|
|
lastFile = output.Name
|
|
}
|
|
lastBytes = output.Bytes
|
|
delta := prevFiles + lastBytes - totalProgress
|
|
totalProgress = bar.Add64(delta)
|
|
}
|
|
|
|
if progress {
|
|
bar.Update()
|
|
}
|
|
case size := <-sizeChan:
|
|
if progress {
|
|
bar.Total = size
|
|
bar.ShowPercent = true
|
|
bar.ShowBar = true
|
|
bar.ShowTimeLeft = true
|
|
}
|
|
case <-req.Context.Done():
|
|
// don't set or print error here, that happens in the goroutine below
|
|
return
|
|
}
|
|
}
|
|
|
|
if progress && bar.Total == 0 && bar.Get() != 0 {
|
|
bar.Total = bar.Get()
|
|
bar.ShowPercent = true
|
|
bar.ShowBar = true
|
|
bar.ShowTimeLeft = true
|
|
bar.Update()
|
|
}
|
|
}
|
|
|
|
if e := res.Error(); e != nil {
|
|
close(outChan)
|
|
return e
|
|
}
|
|
|
|
wait := make(chan struct{})
|
|
go progressBar(wait)
|
|
|
|
defer func() { <-wait }()
|
|
defer close(outChan)
|
|
|
|
for {
|
|
v, err := res.Next()
|
|
if err != nil {
|
|
if err == io.EOF {
|
|
return nil
|
|
}
|
|
|
|
return err
|
|
}
|
|
|
|
select {
|
|
case outChan <- v:
|
|
case <-req.Context.Done():
|
|
return req.Context.Err()
|
|
}
|
|
}
|
|
},
|
|
},
|
|
Type: AddEvent{},
|
|
}
|