From 79b388c690afec7d25813caf945c6ab27e4b6ee3 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Fri, 5 Jan 2018 17:52:58 +0100 Subject: [PATCH 1/4] Feat: use datastore.DiskUsage() and add --size-only to "repo stat" This makes use of the PersistentDatastore DiskUsage method to obtain the Repo's storage usage (GetStorageUsage()). Additionally, the --size-only flag has been added to the "ipfs repo stat" command. This avoids counting the number of objects in the repository and returns faster. License: MIT Signed-off-by: Hector Sanjuan --- core/commands/repo.go | 39 ++++++++++++++++++++++++++++----------- core/corerepo/stat.go | 34 +++++++++++++++++++++++++--------- repo/fsrepo/fsrepo.go | 25 ++----------------------- 3 files changed, 55 insertions(+), 43 deletions(-) diff --git a/core/commands/repo.go b/core/commands/repo.go index c93c42c92..0bd90077a 100644 --- a/core/commands/repo.go +++ b/core/commands/repo.go @@ -150,14 +150,20 @@ var repoStatCmd = &cmds.Command{ Helptext: cmdkit.HelpText{ Tagline: "Get stats for the currently used repo.", ShortDescription: ` -'ipfs repo stat' is a plumbing command that will scan the local -set of stored objects and print repo statistics. It outputs to stdout: +'ipfs repo stat' provides information about the local set of +stored objects. It outputs: + +RepoSize int Size in bytes that the repo is currently taking. +StorageMax string Maximum datastore size (from configuration) NumObjects int Number of objects in the local repo. RepoPath string The path to the repo being currently used. -RepoSize int Size in bytes that the repo is currently taking. Version string The repo version. `, }, + Options: []cmdkit.Option{ + cmdkit.BoolOption("size-only", "Only report RepoSize and StorageMax."), + cmdkit.BoolOption("human", "Output sizes in MiB."), + }, Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) { n, err := GetNode(env) if err != nil { @@ -165,7 +171,14 @@ Version string The repo version. return } - stat, err := corerepo.RepoStat(n, req.Context) + statF := corerepo.RepoStat + + sizeOnly, _ := req.Options["size-only"].(bool) + if sizeOnly { + statF = corerepo.RepoSize + } + + stat, err := statF(req.Context, n) if err != nil { res.SetError(err, cmdkit.ErrNormal) return @@ -173,9 +186,6 @@ Version string The repo version. cmds.EmitOnce(res, stat) }, - Options: []cmdkit.Option{ - cmdkit.BoolOption("human", "Output RepoSize in MiB."), - }, Type: corerepo.Stat{}, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeEncoder(func(req *cmds.Request, w io.Writer, v interface{}) error { @@ -184,17 +194,19 @@ Version string The repo version. return e.TypeErr(stat, v) } - human, _ := req.Options["human"].(bool) - wtr := tabwriter.NewWriter(w, 0, 0, 1, ' ', 0) + defer wtr.Flush() + + human, _ := req.Options["human"].(bool) + sizeOnly, _ := req.Options["size-only"].(bool) - fmt.Fprintf(wtr, "NumObjects:\t%d\n", stat.NumObjects) sizeInMiB := stat.RepoSize / (1024 * 1024) if human && sizeInMiB > 0 { fmt.Fprintf(wtr, "RepoSize (MiB):\t%d\n", sizeInMiB) } else { fmt.Fprintf(wtr, "RepoSize:\t%d\n", stat.RepoSize) } + if stat.StorageMax != corerepo.NoLimit { maxSizeInMiB := stat.StorageMax / (1024 * 1024) if human && maxSizeInMiB > 0 { @@ -203,9 +215,14 @@ Version string The repo version. fmt.Fprintf(wtr, "StorageMax:\t%d\n", stat.StorageMax) } } + + if sizeOnly { + return nil + } + + fmt.Fprintf(wtr, "NumObjects:\t%d\n", stat.NumObjects) fmt.Fprintf(wtr, "RepoPath:\t%s\n", stat.RepoPath) fmt.Fprintf(wtr, "Version:\t%s\n", stat.Version) - wtr.Flush() return nil diff --git a/core/corerepo/stat.go b/core/corerepo/stat.go index 9f756209a..77668ed0f 100644 --- a/core/corerepo/stat.go +++ b/core/corerepo/stat.go @@ -5,27 +5,28 @@ import ( "math" context "context" + "github.com/ipfs/go-ipfs/core" fsrepo "github.com/ipfs/go-ipfs/repo/fsrepo" humanize "gx/ipfs/QmPSBJL4momYnE7DcUyk2DVhD6rH488ZmHBGLbxNdhU44K/go-humanize" ) +// Stat wraps information about the objects stored on disk. type Stat struct { - NumObjects uint64 RepoSize uint64 // size in bytes + StorageMax uint64 // size in bytes + NumObjects uint64 RepoPath string Version string - StorageMax uint64 // size in bytes } // NoLimit represents the value for unlimited storage const NoLimit uint64 = math.MaxUint64 -func RepoStat(n *core.IpfsNode, ctx context.Context) (*Stat, error) { - r := n.Repo - - usage, err := r.GetStorageUsage() +// RepoStat returns a *Stat object with all the fields set. +func RepoStat(ctx context.Context, n *core.IpfsNode) (*Stat, error) { + sizeStat, err := RepoSize(ctx, n) if err != nil { return nil, err } @@ -45,11 +46,29 @@ func RepoStat(n *core.IpfsNode, ctx context.Context) (*Stat, error) { return nil, err } + return &Stat{ + NumObjects: count, + RepoSize: sizeStat.RepoSize, + StorageMax: sizeStat.StorageMax, + RepoPath: path, + Version: fmt.Sprintf("fs-repo@%d", fsrepo.RepoVersion), + }, nil +} + +// RepoSize returns a *Stat object with the RepoSize and StorageMax fields set. +func RepoSize(ctx context.Context, n *core.IpfsNode) (*Stat, error) { + r := n.Repo + cfg, err := r.Config() if err != nil { return nil, err } + usage, err := r.GetStorageUsage() + if err != nil { + return nil, err + } + storageMax := NoLimit if cfg.Datastore.StorageMax != "" { storageMax, err = humanize.ParseBytes(cfg.Datastore.StorageMax) @@ -59,10 +78,7 @@ func RepoStat(n *core.IpfsNode, ctx context.Context) (*Stat, error) { } return &Stat{ - NumObjects: count, RepoSize: usage, - RepoPath: path, - Version: fmt.Sprintf("fs-repo@%d", fsrepo.RepoVersion), StorageMax: storageMax, }, nil } diff --git a/repo/fsrepo/fsrepo.go b/repo/fsrepo/fsrepo.go index 7d53e2263..74075a18a 100644 --- a/repo/fsrepo/fsrepo.go +++ b/repo/fsrepo/fsrepo.go @@ -27,6 +27,7 @@ import ( ma "gx/ipfs/QmYmsdtJ3HsodkePE3eU3TsCaP2YvPZJ4LoXnNkDE5Tpt7/go-multiaddr" lockfile "gx/ipfs/QmYzCZUe9CBDkyPNPcRNqXQK8KKhtUfXvc88PkFujAEJPe/go-fs-lock" logging "gx/ipfs/QmcVVHfdyv15GVPk7NrxdWjh2hLVccXnoD8j2tyQShiXJb/go-log" + ds "gx/ipfs/QmeiCcJfDW1GJnWUArudsv5rQsihpi4oyddPhdqo3CfX6i/go-datastore" ) // LockFile is the filename of the repo lock, relative to config dir @@ -672,29 +673,7 @@ func (r *FSRepo) Datastore() repo.Datastore { // GetStorageUsage computes the storage space taken by the repo in bytes func (r *FSRepo) GetStorageUsage() (uint64, error) { - pth, err := config.PathRoot() - if err != nil { - return 0, err - } - - pth, err = filepath.EvalSymlinks(pth) - if err != nil { - log.Debugf("filepath.EvalSymlinks error: %s", err) - return 0, err - } - - var du uint64 - err = filepath.Walk(pth, func(p string, f os.FileInfo, err error) error { - if err != nil { - log.Debugf("filepath.Walk error: %s", err) - return nil - } - if f != nil { - du += uint64(f.Size()) - } - return nil - }) - return du, err + return ds.DiskUsage(r.Datastore()) } func (r *FSRepo) SwarmKey() ([]byte, error) { From 37d498964739cd84c025dd59cbc6decae71fb33c Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Thu, 10 May 2018 01:22:53 +0200 Subject: [PATCH 2/4] Sharness: add tests for ipfs repo stat --size-only License: MIT Signed-off-by: Hector Sanjuan --- test/sharness/t0080-repo.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/test/sharness/t0080-repo.sh b/test/sharness/t0080-repo.sh index 175069b19..25c5ce714 100755 --- a/test/sharness/t0080-repo.sh +++ b/test/sharness/t0080-repo.sh @@ -245,6 +245,18 @@ test_expect_success "repo stats are updated correctly" ' test $(get_field_num "RepoSize" repo-stats-2) -ge $(get_field_num "RepoSize" repo-stats) ' +test_expect_success "'ipfs repo stat --size-only' succeeds" ' + ipfs repo stat --size-only > repo-stats-size-only +' + +test_expect_success "repo stats came out correct for --size-only" ' + grep "RepoSize" repo-stats-size-only && + grep "StorageMax" repo-stats-size-only && + grep -v "RepoPath" repo-stats-size-only && + grep -v "NumObjects" repo-stats-size-only && + grep -v "Version" repo-stats-size-only +' + test_expect_success "'ipfs repo version' succeeds" ' ipfs repo version > repo-version ' From 9d88d2cde87f10f5d14a8f11c78e4d3d4f5623fe Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Thu, 14 Jun 2018 13:12:45 +0200 Subject: [PATCH 3/4] RepoStat: address review comments License: MIT Signed-off-by: Hector Sanjuan --- core/commands/repo.go | 53 +++++++++++++++++++++++-------------------- core/corerepo/stat.go | 35 ++++++++++++++++------------ 2 files changed, 49 insertions(+), 39 deletions(-) diff --git a/core/commands/repo.go b/core/commands/repo.go index 0bd90077a..5b3ca9e46 100644 --- a/core/commands/repo.go +++ b/core/commands/repo.go @@ -150,7 +150,7 @@ var repoStatCmd = &cmds.Command{ Helptext: cmdkit.HelpText{ Tagline: "Get stats for the currently used repo.", ShortDescription: ` -'ipfs repo stat' provides information about the local set of +'ipfs repo stat' provides information about the local set of stored objects. It outputs: RepoSize int Size in bytes that the repo is currently taking. @@ -171,26 +171,33 @@ Version string The repo version. return } - statF := corerepo.RepoStat - sizeOnly, _ := req.Options["size-only"].(bool) if sizeOnly { - statF = corerepo.RepoSize + sizeStat, err := corerepo.RepoSize(req.Context, n) + if err != nil { + res.SetError(err, cmdkit.ErrNormal) + return + } + cmds.EmitOnce(res, &corerepo.Stat{ + SizeStat: sizeStat, + }) + return } - stat, err := statF(req.Context, n) + stat, err := corerepo.RepoStat(req.Context, n) if err != nil { res.SetError(err, cmdkit.ErrNormal) return } - cmds.EmitOnce(res, stat) + cmds.EmitOnce(res, &stat) }, - Type: corerepo.Stat{}, + Type: &corerepo.Stat{}, Encoders: cmds.EncoderMap{ cmds.Text: cmds.MakeEncoder(func(req *cmds.Request, w io.Writer, v interface{}) error { stat, ok := v.(*corerepo.Stat) if !ok { + fmt.Println("adios") return e.TypeErr(stat, v) } @@ -200,32 +207,28 @@ Version string The repo version. human, _ := req.Options["human"].(bool) sizeOnly, _ := req.Options["size-only"].(bool) - sizeInMiB := stat.RepoSize / (1024 * 1024) - if human && sizeInMiB > 0 { - fmt.Fprintf(wtr, "RepoSize (MiB):\t%d\n", sizeInMiB) - } else { - fmt.Fprintf(wtr, "RepoSize:\t%d\n", stat.RepoSize) - } - - if stat.StorageMax != corerepo.NoLimit { - maxSizeInMiB := stat.StorageMax / (1024 * 1024) - if human && maxSizeInMiB > 0 { - fmt.Fprintf(wtr, "StorageMax (MiB):\t%d\n", maxSizeInMiB) + printSize := func(name string, size uint64) { + sizeInMiB := size / (1024 * 1024) + if human && sizeInMiB > 0 { + fmt.Fprintf(wtr, "%s (MiB):\t%d\n", name, sizeInMiB) } else { - fmt.Fprintf(wtr, "StorageMax:\t%d\n", stat.StorageMax) + fmt.Fprintf(wtr, "%s:\t%d\n", name, size) } } - if sizeOnly { - return nil + if !sizeOnly { + fmt.Fprintf(wtr, "NumObjects:\t%d\n", stat.NumObjects) } - fmt.Fprintf(wtr, "NumObjects:\t%d\n", stat.NumObjects) - fmt.Fprintf(wtr, "RepoPath:\t%s\n", stat.RepoPath) - fmt.Fprintf(wtr, "Version:\t%s\n", stat.Version) + printSize("RepoSize", stat.RepoSize) + printSize("StorageMax", stat.StorageMax) + + if !sizeOnly { + fmt.Fprintf(wtr, "RepoPath:\t%s\n", stat.RepoPath) + fmt.Fprintf(wtr, "Version:\t%s\n", stat.Version) + } return nil - }), }, } diff --git a/core/corerepo/stat.go b/core/corerepo/stat.go index 77668ed0f..28cfef76a 100644 --- a/core/corerepo/stat.go +++ b/core/corerepo/stat.go @@ -12,10 +12,15 @@ import ( humanize "gx/ipfs/QmPSBJL4momYnE7DcUyk2DVhD6rH488ZmHBGLbxNdhU44K/go-humanize" ) -// Stat wraps information about the objects stored on disk. -type Stat struct { +// SizeStat wraps information about the repository size and its limit. +type SizeStat struct { RepoSize uint64 // size in bytes StorageMax uint64 // size in bytes +} + +// Stat wraps information about the objects stored on disk. +type Stat struct { + SizeStat NumObjects uint64 RepoPath string Version string @@ -25,15 +30,15 @@ type Stat struct { const NoLimit uint64 = math.MaxUint64 // RepoStat returns a *Stat object with all the fields set. -func RepoStat(ctx context.Context, n *core.IpfsNode) (*Stat, error) { +func RepoStat(ctx context.Context, n *core.IpfsNode) (Stat, error) { sizeStat, err := RepoSize(ctx, n) if err != nil { - return nil, err + return Stat{}, err } allKeys, err := n.Blockstore.AllKeysChan(ctx) if err != nil { - return nil, err + return Stat{}, err } count := uint64(0) @@ -43,41 +48,43 @@ func RepoStat(ctx context.Context, n *core.IpfsNode) (*Stat, error) { path, err := fsrepo.BestKnownPath() if err != nil { - return nil, err + return Stat{}, err } - return &Stat{ + return Stat{ + SizeStat: SizeStat{ + RepoSize: sizeStat.RepoSize, + StorageMax: sizeStat.StorageMax, + }, NumObjects: count, - RepoSize: sizeStat.RepoSize, - StorageMax: sizeStat.StorageMax, RepoPath: path, Version: fmt.Sprintf("fs-repo@%d", fsrepo.RepoVersion), }, nil } // RepoSize returns a *Stat object with the RepoSize and StorageMax fields set. -func RepoSize(ctx context.Context, n *core.IpfsNode) (*Stat, error) { +func RepoSize(ctx context.Context, n *core.IpfsNode) (SizeStat, error) { r := n.Repo cfg, err := r.Config() if err != nil { - return nil, err + return SizeStat{}, err } usage, err := r.GetStorageUsage() if err != nil { - return nil, err + return SizeStat{}, err } storageMax := NoLimit if cfg.Datastore.StorageMax != "" { storageMax, err = humanize.ParseBytes(cfg.Datastore.StorageMax) if err != nil { - return nil, err + return SizeStat{}, err } } - return &Stat{ + return SizeStat{ RepoSize: usage, StorageMax: storageMax, }, nil From 9c8d994c7647d9d37191d5f5cdb018dabe6b6ab0 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Fri, 15 Jun 2018 10:37:33 +0200 Subject: [PATCH 4/4] remove debug print statement License: MIT Signed-off-by: Hector Sanjuan --- core/commands/repo.go | 1 - 1 file changed, 1 deletion(-) diff --git a/core/commands/repo.go b/core/commands/repo.go index 5b3ca9e46..c9630448a 100644 --- a/core/commands/repo.go +++ b/core/commands/repo.go @@ -197,7 +197,6 @@ Version string The repo version. cmds.Text: cmds.MakeEncoder(func(req *cmds.Request, w io.Writer, v interface{}) error { stat, ok := v.(*corerepo.Stat) if !ok { - fmt.Println("adios") return e.TypeErr(stat, v) }