diff --git a/.gitattributes b/.gitattributes index d6b38c11d..633d55c3b 100644 --- a/.gitattributes +++ b/.gitattributes @@ -9,6 +9,7 @@ LICENSE text eol=auto *.png binary *.tar binary *.gz binary +*.car binary # Binary assets assets/init-doc/* binary diff --git a/core/commands/commands_test.go b/core/commands/commands_test.go index e34c5070c..ee6fd4577 100644 --- a/core/commands/commands_test.go +++ b/core/commands/commands_test.go @@ -24,6 +24,7 @@ func TestROCommands(t *testing.T) { "/commands", "/dag", "/dag/get", + "/dag/export", "/dag/resolve", "/dns", "/get", @@ -95,6 +96,7 @@ func TestCommands(t *testing.T) { "/config/profile/apply", "/dag", "/dag/get", + "/dag/export", "/dag/put", "/dag/resolve", "/dht", diff --git a/core/commands/dag/dag.go b/core/commands/dag/dag.go index 66d50feb1..31ea85060 100644 --- a/core/commands/dag/dag.go +++ b/core/commands/dag/dag.go @@ -14,16 +14,22 @@ import ( cmds "github.com/ipfs/go-ipfs-cmds" files "github.com/ipfs/go-ipfs-files" ipld "github.com/ipfs/go-ipld-format" + mdag "github.com/ipfs/go-merkledag" ipfspath "github.com/ipfs/go-path" path "github.com/ipfs/interface-go-ipfs-core/path" mh "github.com/multiformats/go-multihash" + + gocar "github.com/ipld/go-car" + //gipfree "github.com/ipld/go-ipld-prime/impl/free" + //gipselector "github.com/ipld/go-ipld-prime/traversal/selector" + //gipselectorbuilder "github.com/ipld/go-ipld-prime/traversal/selector/builder" ) var DagCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Interact with ipld dag objects.", ShortDescription: ` -'ipfs dag' is used for creating and manipulating dag objects. +'ipfs dag' is used for creating and manipulating dag objects/hierarchies. This subcommand is currently an experimental feature, but it is intended to deprecate and replace the existing 'ipfs object' command moving forward. @@ -33,6 +39,7 @@ to deprecate and replace the existing 'ipfs object' command moving forward. "put": DagPutCmd, "get": DagGetCmd, "resolve": DagResolveCmd, + "export": DagExportCmd, }, } @@ -241,3 +248,103 @@ var DagResolveCmd = &cmds.Command{ }, Type: ResolveOutput{}, } + +var DagExportCmd = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "Streams the selected DAG as a .car stream on stdout.", + ShortDescription: ` +'ipfs dag export' fetches a dag and streams it out as a well-formed .car file. +Note that at prsent only single root selections / .car files are supported. +The output of blocks happens in strict DAG-traversal, first-seen, order. +`, + }, + Arguments: []cmds.Argument{ + cmds.StringArg("root", true, false, "Expression evaluting to a single root of a dag to export").EnableStdin(), + }, + Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { + + c, err := cid.Decode(req.Arguments[0]) + if err != nil { + return fmt.Errorf( + "unable to parse root specification (currently only bare CIDs are supported): %s", + err, + ) + } + + // The current interface of go-car is rather suboptimal as it + // only takes a blockstore, instead of accepting a dagservice, + // and leveraging parallel-fetch capabilities + // https://github.com/ipld/go-car/issues/27 + // + // Until the above is fixed, pre-warm the blockstore before doing + // anything else. We explicitly *DO NOT* take a lock during this + // operation: even if we lose some of the blocks we just received + // due to a conflicting GC: we will just re-retrieve anything we + // potentially lost when the car is being streamed out + node, err := cmdenv.GetNode(env) + if err != nil { + return err + } + + if err := mdag.FetchGraph(req.Context, c, node.DAG); err != nil { + if !node.IsOnline { + err = fmt.Errorf("%s (currently offline, perhaps retry after attaching to the network)", err) + } + return err + } + + // Code disabled until descent-issue in go-ipld-prime is fixed + // + // The second part of the above - make a super-thin wrapper around + // a blockservice session, translating Session.GetBlock() to Blockstore.Get() + // + // sess := blockservice.NewSession( + // req.Context, + // node.Blocks, + // ) + // var wrapper getBlockFromSessionWrapper = func(c cid.Cid) (blk.Block, error) { + // return sess.GetBlock(req.Context, c) + // } + // sb := gipselectorbuilder.NewSelectorSpecBuilder(gipfree.NodeBuilder()) + // car := gocar.NewSelectiveCar( + // req.Context, + // &wrapper, + // []gocar.Dag{gocar.Dag{ + // Root: c, + // Selector: sb.ExploreRecursive( + // gipselector.RecursionLimitNone(), + // sb.ExploreAll(sb.ExploreRecursiveEdge()), + // ).Node(), + // }}, + // ) + + pipeR, pipeW := io.Pipe() + + errCh := make(chan error, 2) // we only report the 1st error + go func() { + defer func() { + if err := pipeW.Close(); err != nil { + errCh <- fmt.Errorf("stream flush failed: %s", err) + } + close(errCh) + }() + + //if err := car.Write(pipeW); err != nil { + if err := gocar.WriteCar( + req.Context, + node.DAG, + []cid.Cid{c}, + pipeW, + ); err != nil { + errCh <- err + } + }() + + if err := res.Emit(pipeR); err != nil { + pipeW.Close() // ignore errors if any + return err + } + + return <-errCh + }, +} diff --git a/core/commands/root.go b/core/commands/root.go index 752cfd4e3..c31199308 100644 --- a/core/commands/root.go +++ b/core/commands/root.go @@ -190,6 +190,7 @@ var rootROSubcommands = map[string]*cmds.Command{ "dag": { Subcommands: map[string]*cmds.Command{ "get": dag.DagGetCmd, + "export": dag.DagExportCmd, "resolve": dag.DagResolveCmd, }, }, diff --git a/go.mod b/go.mod index fab16101a..b9b3f513e 100644 --- a/go.mod +++ b/go.mod @@ -54,6 +54,7 @@ require ( github.com/ipfs/go-unixfs v0.2.4 github.com/ipfs/go-verifcid v0.0.1 github.com/ipfs/interface-go-ipfs-core v0.2.6 + github.com/ipld/go-car v0.0.5-0.20200316204026-3e2cf7af0fab github.com/jbenet/go-is-domain v1.0.3 github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c github.com/jbenet/go-temp-err-catcher v0.1.0 diff --git a/go.sum b/go.sum index 0e3cbf128..6de9d5046 100644 --- a/go.sum +++ b/go.sum @@ -18,8 +18,10 @@ github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBA github.com/alangpierce/go-forceexport v0.0.0-20160317203124-8f1d6941cd75 h1:3ILjVyslFbc4jl1w5TWuvvslFD/nDfR2H8tVaMVLrEY= github.com/alangpierce/go-forceexport v0.0.0-20160317203124-8f1d6941cd75/go.mod h1:uAXEEpARkRhCZfEvy/y0Jcc888f9tHCc1W7/UeEtreE= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4 h1:Hs82Z41s6SdL1CELW+XaDYmOH4hkBN4/N9og/AsOv7E= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -190,15 +192,12 @@ github.com/ipfs/go-blockservice v0.0.3/go.mod h1:/NNihwTi6V2Yr6g8wBI+BSwPuURpBRM github.com/ipfs/go-blockservice v0.0.7/go.mod h1:EOfb9k/Y878ZTRY/CH0x5+ATtaipfbRhbvNSdgc/7So= github.com/ipfs/go-blockservice v0.1.0/go.mod h1:hzmMScl1kXHg3M2BjTymbVPjv627N7sYcvYaKbop39M= github.com/ipfs/go-blockservice v0.1.1/go.mod h1:t+411r7psEUhLueM8C7aPA7cxCclv4O3VsUVxt9kz2I= -github.com/ipfs/go-blockservice v0.1.2 h1:fqFeeu1EG0lGVrqUo+BVJv7LZV31I4ZsyNthCOMAJRc= github.com/ipfs/go-blockservice v0.1.2/go.mod h1:t+411r7psEUhLueM8C7aPA7cxCclv4O3VsUVxt9kz2I= github.com/ipfs/go-blockservice v0.1.3 h1:9XgsPMwwWJSC9uVr2pMDsW2qFTBSkxpGMhmna8mIjPM= github.com/ipfs/go-blockservice v0.1.3/go.mod h1:OTZhFpkgY48kNzbgyvcexW9cHrpjBYIjSR0KoDOFOLU= github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= github.com/ipfs/go-cid v0.0.2/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= -github.com/ipfs/go-cid v0.0.3 h1:UIAh32wymBpStoe83YCzwVQQ5Oy/H0FdxvUS6DJDzms= github.com/ipfs/go-cid v0.0.3/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= -github.com/ipfs/go-cid v0.0.4 h1:UlfXKrZx1DjZoBhQHmNHLC1fK1dUJDN20Y28A7s+gJ8= github.com/ipfs/go-cid v0.0.4/go.mod h1:4LLaPOQwmk5z9LBgQnpkivrx8BJjUyGwTXCd5Xfj6+M= github.com/ipfs/go-cid v0.0.5 h1:o0Ix8e/ql7Zb5UVUJEUfjsWCIY8t48++9lR8qi6oiJU= github.com/ipfs/go-cid v0.0.5/go.mod h1:plgt+Y5MnOey4vO4UlUazGqdbEXuFYitED67FexhXog= @@ -326,6 +325,7 @@ github.com/ipfs/go-merkledag v0.0.6/go.mod h1:QYPdnlvkOg7GnQRofu9XZimC5ZW5Wi3bKy github.com/ipfs/go-merkledag v0.1.0/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk= github.com/ipfs/go-merkledag v0.2.3 h1:aMdkK9G1hEeNvn3VXfiEMLY0iJnbiQQUHnM0HFJREsE= github.com/ipfs/go-merkledag v0.2.3/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk= +github.com/ipfs/go-merkledag v0.2.4/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk= github.com/ipfs/go-merkledag v0.3.0 h1:1bXv/ZRPZLVdij/a33CkXMVdxUdred9sz4xyph+0ls0= github.com/ipfs/go-merkledag v0.3.0/go.mod h1:4pymaZLhSLNVuiCITYrpViD6vmfZ/Ws4n/L9tfNv3S4= github.com/ipfs/go-merkledag v0.3.1 h1:3UqWINBEr3/N+r6OwgFXAddDP/8zpQX/8J7IGVOCqRQ= @@ -354,6 +354,8 @@ github.com/ipfs/go-verifcid v0.0.1 h1:m2HI7zIuR5TFyQ1b79Da5N9dnnCP1vcu2QqawmWlK2 github.com/ipfs/go-verifcid v0.0.1/go.mod h1:5Hrva5KBeIog4A+UpqlaIU+DEstipcJYQQZc0g37pY0= github.com/ipfs/interface-go-ipfs-core v0.2.6 h1:4eeGPJUDWblEurSzpAtL2znDG1xqAoX2aNNIoOFwyuc= github.com/ipfs/interface-go-ipfs-core v0.2.6/go.mod h1:Tihp8zxGpUeE3Tokr94L6zWZZdkRQvG5TL6i9MuNE+s= +github.com/ipld/go-car v0.0.5-0.20200316204026-3e2cf7af0fab h1:+3Y6Jb3IBmG3t6e3r6TItnuciOaMOuGW7QIVEUa5vy4= +github.com/ipld/go-car v0.0.5-0.20200316204026-3e2cf7af0fab/go.mod h1:yR5AsJ38xTwwgwGpbh60ICtdLPp5lGfuH28PAAzaEhM= github.com/ipld/go-ipld-prime v0.0.2-0.20191108012745-28a82f04c785 h1:fASnkvtR+SmB2y453RxmDD3Uvd4LonVUgFGk9JoDaZs= github.com/ipld/go-ipld-prime v0.0.2-0.20191108012745-28a82f04c785/go.mod h1:bDDSvVz7vaK12FNvMeRYnpRFkSUPNQOiCYQezMD/P3w= github.com/ipld/go-ipld-prime-proto v0.0.0-20191113031812-e32bd156a1e5 h1:lSip43rAdyGA+yRQuy6ju0ucZkWpYc1F2CTQtZTVW/4= @@ -554,7 +556,6 @@ github.com/libp2p/go-libp2p-peerstore v0.2.0 h1:XcgJhI8WyUOCbHyRLNEX5542YNj8hnLS github.com/libp2p/go-libp2p-peerstore v0.2.0/go.mod h1:N2l3eVIeAitSg3Pi2ipSrJYnqhVnMNQZo9nkSCuAbnQ= github.com/libp2p/go-libp2p-peerstore v0.2.1 h1:u+gOfsKgu73ZkGWhvckRm03z9C+iS9TrLqpANweELGs= github.com/libp2p/go-libp2p-peerstore v0.2.1/go.mod h1:NQxhNjWxf1d4w6PihR8btWIRjwRLBr4TYKfNgrUkOPA= -github.com/libp2p/go-libp2p-peerstore v0.2.2 h1:iqc/m03jHn5doXN3+kS6JKvqQRHEltiXljQB85iVHWE= github.com/libp2p/go-libp2p-peerstore v0.2.2/go.mod h1:NQxhNjWxf1d4w6PihR8btWIRjwRLBr4TYKfNgrUkOPA= github.com/libp2p/go-libp2p-peerstore v0.2.3 h1:MofRq2l3c15vQpEygTetV+zRRrncz+ktiXW7H2EKoEQ= github.com/libp2p/go-libp2p-peerstore v0.2.3/go.mod h1:K8ljLdFn590GMttg/luh4caB/3g0vKuY01psze0upRw= @@ -902,6 +903,7 @@ github.com/texttheater/golang-levenshtein v0.0.0-20180516184445-d188e65d659e/go. github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c h1:u6SKchux2yDvFQnDHS3lPnIRmfVJ5Sxy3ao2SIdysLQ= github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/vishvananda/netlink v1.1.0 h1:1iyaYNBLmP6L0220aDnYQpo1QEV4t4hJ+xEEhhJH8j0= github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df h1:OviZH7qLw/7ZovXvuNyL3XQl8UFofeikI1NW1Gypu7k= diff --git a/test/sharness/t0054-dag-car-import-export.sh b/test/sharness/t0054-dag-car-import-export.sh new file mode 100755 index 000000000..d40dd666f --- /dev/null +++ b/test/sharness/t0054-dag-car-import-export.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# + +test_description="Test car file import/export functionality" + +. lib/test-lib.sh + +test_init_ipfs + + +echo "Error: merkledag: not found (currently offline, perhaps retry after attaching to the network)" > offline_fetch_error_expected + +test_expect_success "basic offline export of nonexistent cid" ' + ! ipfs dag export QmYwAPJXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX 2> offline_fetch_error_actual +' + +test_expect_success "correct error" ' + test_cmp offline_fetch_error_expected offline_fetch_error_actual +' + +test_expect_success "basic offline export of 'getting started' dag" ' + ipfs dag export QmS4ustL54uo8FzR9455qaxZwuMiUhyvMcX9Ba8nUH4uVv >/dev/null +' + +test_done