From 13e79a05e541ecfa71ff2390c78005076ba50e9a Mon Sep 17 00:00:00 2001 From: Juan Batiz-Benet Date: Wed, 7 Jan 2015 02:13:44 -0800 Subject: [PATCH 1/3] merkledag: add NodeStat object --- merkledag/merkledag.go | 35 +++++++++++++++++++++++++++++++++++ merkledag/merkledag_test.go | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+) diff --git a/merkledag/merkledag.go b/merkledag/merkledag.go index cd50d9e5b..bfb7f3ccf 100644 --- a/merkledag/merkledag.go +++ b/merkledag/merkledag.go @@ -51,6 +51,20 @@ type Node struct { cached mh.Multihash } +// NodeStat is a statistics object for a Node. Mostly sizes. +type NodeStat struct { + NumLinks int // number of links in link table + BlockSize int // size of the raw data + LinksSize int // size of the links segment + DataSize int // size of the data segment + CumulativeSize int // cumulatie size of object + all it references +} + +func (ns NodeStat) String() string { + f := "NodeStat{NumLinks: %d, BlockSize: %d, LinksSize: %d, DataSize: %d, CumulativeSize: %d}" + return fmt.Sprintf(f, ns.NumLinks, ns.BlockSize, ns.LinksSize, ns.DataSize, ns.CumulativeSize) +} + // Link represents an IPFS Merkle DAG Link between Nodes. type Link struct { // utf string name. should be unique per object @@ -162,6 +176,27 @@ func (n *Node) Size() (uint64, error) { return s, nil } +// Stat returns statistics on the node. +func (n *Node) Stat() (NodeStat, error) { + enc, err := n.Encoded(false) + if err != nil { + return NodeStat{}, err + } + + cumSize, err := n.Size() + if err != nil { + return NodeStat{}, err + } + + return NodeStat{ + NumLinks: len(n.Links), + BlockSize: len(enc), + LinksSize: len(enc) - len(n.Data), // includes framing. + DataSize: len(n.Data), + CumulativeSize: int(cumSize), + }, nil +} + // Multihash hashes the encoded data of this node. func (n *Node) Multihash() (mh.Multihash, error) { // Note: Encoded generates the hash and puts it in n.cached. diff --git a/merkledag/merkledag_test.go b/merkledag/merkledag_test.go index 0c5bf71a8..e44870b8c 100644 --- a/merkledag/merkledag_test.go +++ b/merkledag/merkledag_test.go @@ -85,6 +85,8 @@ func TestNode(t *testing.T) { } else { fmt.Println("key: ", k) } + + SubtestNodeStat(t, n) } printn("beep", n1) @@ -92,6 +94,40 @@ func TestNode(t *testing.T) { printn("beep boop", n3) } +func SubtestNodeStat(t *testing.T, n *Node) { + enc, err := n.Encoded(true) + if err != nil { + t.Error("n.Encoded(true) failed") + return + } + + cumSize, err := n.Size() + if err != nil { + t.Error("n.Size() failed") + return + } + + expected := NodeStat{ + NumLinks: len(n.Links), + BlockSize: len(enc), + LinksSize: len(enc) - len(n.Data), // includes framing. + DataSize: len(n.Data), + CumulativeSize: int(cumSize), + } + + actual, err := n.Stat() + if err != nil { + t.Error("n.Stat() failed") + return + } + + if expected != actual { + t.Error("n.Stat incorrect.\nexpect: %s\nactual: %s", expected, actual) + } else { + fmt.Printf("n.Stat correct: %s\n", actual) + } +} + type devZero struct{} func (_ devZero) Read(b []byte) (int, error) { From f63d45ca56733c4b899ab97483ebc01fbd592770 Mon Sep 17 00:00:00 2001 From: Juan Batiz-Benet Date: Wed, 7 Jan 2015 02:15:53 -0800 Subject: [PATCH 2/3] merkledag: split off node.go --- merkledag/merkledag.go | 181 --------------------------------------- merkledag/node.go | 188 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 188 insertions(+), 181 deletions(-) create mode 100644 merkledag/node.go diff --git a/merkledag/merkledag.go b/merkledag/merkledag.go index bfb7f3ccf..16427c484 100644 --- a/merkledag/merkledag.go +++ b/merkledag/merkledag.go @@ -9,7 +9,6 @@ import ( "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context" - mh "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multihash" blocks "github.com/jbenet/go-ipfs/blocks" bserv "github.com/jbenet/go-ipfs/blockservice" u "github.com/jbenet/go-ipfs/util" @@ -18,11 +17,6 @@ import ( var log = u.Logger("merkledag") var ErrNotFound = fmt.Errorf("merkledag: not found") -// NodeMap maps u.Keys to Nodes. -// We cannot use []byte/Multihash for keys :( -// so have to convert Multihash bytes to string (u.Key) -type NodeMap map[u.Key]*Node - // DAGService is an IPFS Merkle DAG service. type DAGService interface { Add(*Node) (u.Key, error) @@ -39,181 +33,6 @@ func NewDAGService(bs *bserv.BlockService) DAGService { return &dagService{bs} } -// Node represents a node in the IPFS Merkle DAG. -// nodes have opaque data and a set of navigable links. -type Node struct { - Links []*Link - Data []byte - - // cache encoded/marshaled value - encoded []byte - - cached mh.Multihash -} - -// NodeStat is a statistics object for a Node. Mostly sizes. -type NodeStat struct { - NumLinks int // number of links in link table - BlockSize int // size of the raw data - LinksSize int // size of the links segment - DataSize int // size of the data segment - CumulativeSize int // cumulatie size of object + all it references -} - -func (ns NodeStat) String() string { - f := "NodeStat{NumLinks: %d, BlockSize: %d, LinksSize: %d, DataSize: %d, CumulativeSize: %d}" - return fmt.Sprintf(f, ns.NumLinks, ns.BlockSize, ns.LinksSize, ns.DataSize, ns.CumulativeSize) -} - -// Link represents an IPFS Merkle DAG Link between Nodes. -type Link struct { - // utf string name. should be unique per object - Name string // utf8 - - // cumulative size of target object - Size uint64 - - // multihash of the target object - Hash mh.Multihash - - // a ptr to the actual node for graph manipulation - Node *Node -} - -type LinkSlice []*Link - -func (ls LinkSlice) Len() int { return len(ls) } -func (ls LinkSlice) Swap(a, b int) { ls[a], ls[b] = ls[b], ls[a] } -func (ls LinkSlice) Less(a, b int) bool { return ls[a].Name < ls[b].Name } - -// MakeLink creates a link to the given node -func MakeLink(n *Node) (*Link, error) { - s, err := n.Size() - if err != nil { - return nil, err - } - - h, err := n.Multihash() - if err != nil { - return nil, err - } - return &Link{ - Size: s, - Hash: h, - }, nil -} - -// GetNode returns the MDAG Node that this link points to -func (l *Link) GetNode(serv DAGService) (*Node, error) { - if l.Node != nil { - return l.Node, nil - } - - return serv.Get(u.Key(l.Hash)) -} - -// AddNodeLink adds a link to another node. -func (n *Node) AddNodeLink(name string, that *Node) error { - lnk, err := MakeLink(that) - if err != nil { - return err - } - lnk.Name = name - lnk.Node = that - - n.Links = append(n.Links, lnk) - return nil -} - -// AddNodeLink adds a link to another node. without keeping a reference to -// the child node -func (n *Node) AddNodeLinkClean(name string, that *Node) error { - lnk, err := MakeLink(that) - if err != nil { - return err - } - lnk.Name = name - - n.Links = append(n.Links, lnk) - return nil -} - -// Remove a link on this node by the given name -func (n *Node) RemoveNodeLink(name string) error { - for i, l := range n.Links { - if l.Name == name { - n.Links = append(n.Links[:i], n.Links[i+1:]...) - return nil - } - } - return ErrNotFound -} - -// Copy returns a copy of the node. -// NOTE: does not make copies of Node objects in the links. -func (n *Node) Copy() *Node { - nnode := new(Node) - nnode.Data = make([]byte, len(n.Data)) - copy(nnode.Data, n.Data) - - nnode.Links = make([]*Link, len(n.Links)) - copy(nnode.Links, n.Links) - return nnode -} - -// Size returns the total size of the data addressed by node, -// including the total sizes of references. -func (n *Node) Size() (uint64, error) { - b, err := n.Encoded(false) - if err != nil { - return 0, err - } - - s := uint64(len(b)) - for _, l := range n.Links { - s += l.Size - } - return s, nil -} - -// Stat returns statistics on the node. -func (n *Node) Stat() (NodeStat, error) { - enc, err := n.Encoded(false) - if err != nil { - return NodeStat{}, err - } - - cumSize, err := n.Size() - if err != nil { - return NodeStat{}, err - } - - return NodeStat{ - NumLinks: len(n.Links), - BlockSize: len(enc), - LinksSize: len(enc) - len(n.Data), // includes framing. - DataSize: len(n.Data), - CumulativeSize: int(cumSize), - }, nil -} - -// Multihash hashes the encoded data of this node. -func (n *Node) Multihash() (mh.Multihash, error) { - // Note: Encoded generates the hash and puts it in n.cached. - _, err := n.Encoded(false) - if err != nil { - return nil, err - } - - return n.cached, nil -} - -// Key returns the Multihash as a key, for maps. -func (n *Node) Key() (u.Key, error) { - h, err := n.Multihash() - return u.Key(h), err -} - // dagService is an IPFS Merkle DAG service. // - the root is virtual (like a forest) // - stores nodes' data in a BlockService diff --git a/merkledag/node.go b/merkledag/node.go new file mode 100644 index 000000000..2f522ce54 --- /dev/null +++ b/merkledag/node.go @@ -0,0 +1,188 @@ +package merkledag + +import ( + "fmt" + + mh "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multihash" + u "github.com/jbenet/go-ipfs/util" +) + +// NodeMap maps u.Keys to Nodes. +// We cannot use []byte/Multihash for keys :( +// so have to convert Multihash bytes to string (u.Key) +type NodeMap map[u.Key]*Node + +// Node represents a node in the IPFS Merkle DAG. +// nodes have opaque data and a set of navigable links. +type Node struct { + Links []*Link + Data []byte + + // cache encoded/marshaled value + encoded []byte + + cached mh.Multihash +} + +// NodeStat is a statistics object for a Node. Mostly sizes. +type NodeStat struct { + NumLinks int // number of links in link table + BlockSize int // size of the raw data + LinksSize int // size of the links segment + DataSize int // size of the data segment + CumulativeSize int // cumulatie size of object + all it references +} + +func (ns NodeStat) String() string { + f := "NodeStat{NumLinks: %d, BlockSize: %d, LinksSize: %d, DataSize: %d, CumulativeSize: %d}" + return fmt.Sprintf(f, ns.NumLinks, ns.BlockSize, ns.LinksSize, ns.DataSize, ns.CumulativeSize) +} + +// Link represents an IPFS Merkle DAG Link between Nodes. +type Link struct { + // utf string name. should be unique per object + Name string // utf8 + + // cumulative size of target object + Size uint64 + + // multihash of the target object + Hash mh.Multihash + + // a ptr to the actual node for graph manipulation + Node *Node +} + +type LinkSlice []*Link + +func (ls LinkSlice) Len() int { return len(ls) } +func (ls LinkSlice) Swap(a, b int) { ls[a], ls[b] = ls[b], ls[a] } +func (ls LinkSlice) Less(a, b int) bool { return ls[a].Name < ls[b].Name } + +// MakeLink creates a link to the given node +func MakeLink(n *Node) (*Link, error) { + s, err := n.Size() + if err != nil { + return nil, err + } + + h, err := n.Multihash() + if err != nil { + return nil, err + } + return &Link{ + Size: s, + Hash: h, + }, nil +} + +// GetNode returns the MDAG Node that this link points to +func (l *Link) GetNode(serv DAGService) (*Node, error) { + if l.Node != nil { + return l.Node, nil + } + + return serv.Get(u.Key(l.Hash)) +} + +// AddNodeLink adds a link to another node. +func (n *Node) AddNodeLink(name string, that *Node) error { + lnk, err := MakeLink(that) + if err != nil { + return err + } + lnk.Name = name + lnk.Node = that + + n.Links = append(n.Links, lnk) + return nil +} + +// AddNodeLink adds a link to another node. without keeping a reference to +// the child node +func (n *Node) AddNodeLinkClean(name string, that *Node) error { + lnk, err := MakeLink(that) + if err != nil { + return err + } + lnk.Name = name + + n.Links = append(n.Links, lnk) + return nil +} + +// Remove a link on this node by the given name +func (n *Node) RemoveNodeLink(name string) error { + for i, l := range n.Links { + if l.Name == name { + n.Links = append(n.Links[:i], n.Links[i+1:]...) + return nil + } + } + return ErrNotFound +} + +// Copy returns a copy of the node. +// NOTE: does not make copies of Node objects in the links. +func (n *Node) Copy() *Node { + nnode := new(Node) + nnode.Data = make([]byte, len(n.Data)) + copy(nnode.Data, n.Data) + + nnode.Links = make([]*Link, len(n.Links)) + copy(nnode.Links, n.Links) + return nnode +} + +// Size returns the total size of the data addressed by node, +// including the total sizes of references. +func (n *Node) Size() (uint64, error) { + b, err := n.Encoded(false) + if err != nil { + return 0, err + } + + s := uint64(len(b)) + for _, l := range n.Links { + s += l.Size + } + return s, nil +} + +// Stat returns statistics on the node. +func (n *Node) Stat() (NodeStat, error) { + enc, err := n.Encoded(false) + if err != nil { + return NodeStat{}, err + } + + cumSize, err := n.Size() + if err != nil { + return NodeStat{}, err + } + + return NodeStat{ + NumLinks: len(n.Links), + BlockSize: len(enc), + LinksSize: len(enc) - len(n.Data), // includes framing. + DataSize: len(n.Data), + CumulativeSize: int(cumSize), + }, nil +} + +// Multihash hashes the encoded data of this node. +func (n *Node) Multihash() (mh.Multihash, error) { + // Note: Encoded generates the hash and puts it in n.cached. + _, err := n.Encoded(false) + if err != nil { + return nil, err + } + + return n.cached, nil +} + +// Key returns the Multihash as a key, for maps. +func (n *Node) Key() (u.Key, error) { + h, err := n.Multihash() + return u.Key(h), err +} From 6b2795338f918c9e6bf1ca4dc864b50fcaa7bafb Mon Sep 17 00:00:00 2001 From: Juan Batiz-Benet Date: Wed, 7 Jan 2015 02:30:02 -0800 Subject: [PATCH 3/3] ipfs object has learned stat 'ipfs object stat' is a plumbing command to print DAG node statistics. is a base58 encoded multihash. It outputs to stdout: NumLinks int number of links in link table BlockSize int size of the raw, encoded data LinksSize int size of the links segment DataSize int size of the data segment CumulativeSize int cumulative size of object and references --- core/commands/object.go | 61 +++++++++++++++++++++++++++++++++++++++++ merkledag/node.go | 4 +-- 2 files changed, 63 insertions(+), 2 deletions(-) diff --git a/core/commands/object.go b/core/commands/object.go index 19d02795f..055a7f67d 100644 --- a/core/commands/object.go +++ b/core/commands/object.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/json" "errors" + "fmt" "io" "io/ioutil" "strings" @@ -36,6 +37,7 @@ ipfs object get - Get the DAG node named by ipfs object put - Stores input, outputs its key ipfs object data - Outputs raw bytes in an object ipfs object links - Outputs links pointed to by object +ipfs object stat - Outputs statistics of object `, }, @@ -44,6 +46,7 @@ ipfs object links - Outputs links pointed to by object "links": objectLinksCmd, "get": objectGetCmd, "put": objectPutCmd, + "stat": objectStatCmd, }, } @@ -180,6 +183,64 @@ This command outputs data in the following encodings: }, } +var objectStatCmd = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "Get stats for the DAG node named by ", + ShortDescription: ` +'ipfs object stat' is a plumbing command to print DAG node statistics. + is a base58 encoded multihash. It outputs to stdout: + + NumLinks int number of links in link table + BlockSize int size of the raw, encoded data + LinksSize int size of the links segment + DataSize int size of the data segment + CumulativeSize int cumulative size of object and its references +`, + }, + + Arguments: []cmds.Argument{ + cmds.StringArg("key", true, false, "Key of the object to retrieve (in base58-encoded multihash format)"), + }, + Run: func(req cmds.Request) (interface{}, error) { + n, err := req.Context().GetNode() + if err != nil { + return nil, err + } + + key := req.Arguments()[0] + + object, err := objectGet(n, key) + if err != nil { + return nil, err + } + + ns, err := object.Stat() + if err != nil { + return nil, err + } + + return ns, nil + }, + Type: dag.NodeStat{}, + Marshalers: cmds.MarshalerMap{ + cmds.Text: func(res cmds.Response) (io.Reader, error) { + ns := res.Output().(dag.NodeStat) + + var buf bytes.Buffer + w := func(s string, n int) { + buf.Write([]byte(fmt.Sprintf("%s: %d\n", s, n))) + } + w("NumLinks", ns.NumLinks) + w("BlockSize", ns.BlockSize) + w("LinksSize", ns.LinksSize) + w("DataSize", ns.DataSize) + w("CumulativeSize", ns.CumulativeSize) + + return &buf, nil + }, + }, +} + var objectPutCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Stores input as a DAG object, outputs its key", diff --git a/merkledag/node.go b/merkledag/node.go index 2f522ce54..016f4353b 100644 --- a/merkledag/node.go +++ b/merkledag/node.go @@ -27,10 +27,10 @@ type Node struct { // NodeStat is a statistics object for a Node. Mostly sizes. type NodeStat struct { NumLinks int // number of links in link table - BlockSize int // size of the raw data + BlockSize int // size of the raw, encoded data LinksSize int // size of the links segment DataSize int // size of the data segment - CumulativeSize int // cumulatie size of object + all it references + CumulativeSize int // cumulative size of object and its references } func (ns NodeStat) String() string {