mirror of
https://github.com/ipfs/kubo.git
synced 2025-06-01 16:05:47 +08:00
Dag export command, silent, no progress
This commit is contained in:
Peter Rabbitson
committed by
Steven Allen
Steven Allen
parent
23af25f176
commit
635e333d60
1
.gitattributes
vendored
1
.gitattributes
vendored
@ -9,6 +9,7 @@ LICENSE text eol=auto
|
||||
*.png binary
|
||||
*.tar binary
|
||||
*.gz binary
|
||||
*.car binary
|
||||
|
||||
# Binary assets
|
||||
assets/init-doc/* binary
|
||||
|
||||
@ -24,6 +24,7 @@ func TestROCommands(t *testing.T) {
|
||||
"/commands",
|
||||
"/dag",
|
||||
"/dag/get",
|
||||
"/dag/export",
|
||||
"/dag/resolve",
|
||||
"/dns",
|
||||
"/get",
|
||||
@ -95,6 +96,7 @@ func TestCommands(t *testing.T) {
|
||||
"/config/profile/apply",
|
||||
"/dag",
|
||||
"/dag/get",
|
||||
"/dag/export",
|
||||
"/dag/put",
|
||||
"/dag/resolve",
|
||||
"/dht",
|
||||
|
||||
@ -14,16 +14,22 @@ import (
|
||||
cmds "github.com/ipfs/go-ipfs-cmds"
|
||||
files "github.com/ipfs/go-ipfs-files"
|
||||
ipld "github.com/ipfs/go-ipld-format"
|
||||
mdag "github.com/ipfs/go-merkledag"
|
||||
ipfspath "github.com/ipfs/go-path"
|
||||
path "github.com/ipfs/interface-go-ipfs-core/path"
|
||||
mh "github.com/multiformats/go-multihash"
|
||||
|
||||
gocar "github.com/ipld/go-car"
|
||||
//gipfree "github.com/ipld/go-ipld-prime/impl/free"
|
||||
//gipselector "github.com/ipld/go-ipld-prime/traversal/selector"
|
||||
//gipselectorbuilder "github.com/ipld/go-ipld-prime/traversal/selector/builder"
|
||||
)
|
||||
|
||||
var DagCmd = &cmds.Command{
|
||||
Helptext: cmds.HelpText{
|
||||
Tagline: "Interact with ipld dag objects.",
|
||||
ShortDescription: `
|
||||
'ipfs dag' is used for creating and manipulating dag objects.
|
||||
'ipfs dag' is used for creating and manipulating dag objects/hierarchies.
|
||||
|
||||
This subcommand is currently an experimental feature, but it is intended
|
||||
to deprecate and replace the existing 'ipfs object' command moving forward.
|
||||
@ -33,6 +39,7 @@ to deprecate and replace the existing 'ipfs object' command moving forward.
|
||||
"put": DagPutCmd,
|
||||
"get": DagGetCmd,
|
||||
"resolve": DagResolveCmd,
|
||||
"export": DagExportCmd,
|
||||
},
|
||||
}
|
||||
|
||||
@ -241,3 +248,103 @@ var DagResolveCmd = &cmds.Command{
|
||||
},
|
||||
Type: ResolveOutput{},
|
||||
}
|
||||
|
||||
var DagExportCmd = &cmds.Command{
|
||||
Helptext: cmds.HelpText{
|
||||
Tagline: "Streams the selected DAG as a .car stream on stdout.",
|
||||
ShortDescription: `
|
||||
'ipfs dag export' fetches a dag and streams it out as a well-formed .car file.
|
||||
Note that at prsent only single root selections / .car files are supported.
|
||||
The output of blocks happens in strict DAG-traversal, first-seen, order.
|
||||
`,
|
||||
},
|
||||
Arguments: []cmds.Argument{
|
||||
cmds.StringArg("root", true, false, "Expression evaluting to a single root of a dag to export").EnableStdin(),
|
||||
},
|
||||
Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
|
||||
|
||||
c, err := cid.Decode(req.Arguments[0])
|
||||
if err != nil {
|
||||
return fmt.Errorf(
|
||||
"unable to parse root specification (currently only bare CIDs are supported): %s",
|
||||
err,
|
||||
)
|
||||
}
|
||||
|
||||
// The current interface of go-car is rather suboptimal as it
|
||||
// only takes a blockstore, instead of accepting a dagservice,
|
||||
// and leveraging parallel-fetch capabilities
|
||||
// https://github.com/ipld/go-car/issues/27
|
||||
//
|
||||
// Until the above is fixed, pre-warm the blockstore before doing
|
||||
// anything else. We explicitly *DO NOT* take a lock during this
|
||||
// operation: even if we lose some of the blocks we just received
|
||||
// due to a conflicting GC: we will just re-retrieve anything we
|
||||
// potentially lost when the car is being streamed out
|
||||
node, err := cmdenv.GetNode(env)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := mdag.FetchGraph(req.Context, c, node.DAG); err != nil {
|
||||
if !node.IsOnline {
|
||||
err = fmt.Errorf("%s (currently offline, perhaps retry after attaching to the network)", err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// Code disabled until descent-issue in go-ipld-prime is fixed
|
||||
//
|
||||
// The second part of the above - make a super-thin wrapper around
|
||||
// a blockservice session, translating Session.GetBlock() to Blockstore.Get()
|
||||
//
|
||||
// sess := blockservice.NewSession(
|
||||
// req.Context,
|
||||
// node.Blocks,
|
||||
// )
|
||||
// var wrapper getBlockFromSessionWrapper = func(c cid.Cid) (blk.Block, error) {
|
||||
// return sess.GetBlock(req.Context, c)
|
||||
// }
|
||||
// sb := gipselectorbuilder.NewSelectorSpecBuilder(gipfree.NodeBuilder())
|
||||
// car := gocar.NewSelectiveCar(
|
||||
// req.Context,
|
||||
// &wrapper,
|
||||
// []gocar.Dag{gocar.Dag{
|
||||
// Root: c,
|
||||
// Selector: sb.ExploreRecursive(
|
||||
// gipselector.RecursionLimitNone(),
|
||||
// sb.ExploreAll(sb.ExploreRecursiveEdge()),
|
||||
// ).Node(),
|
||||
// }},
|
||||
// )
|
||||
|
||||
pipeR, pipeW := io.Pipe()
|
||||
|
||||
errCh := make(chan error, 2) // we only report the 1st error
|
||||
go func() {
|
||||
defer func() {
|
||||
if err := pipeW.Close(); err != nil {
|
||||
errCh <- fmt.Errorf("stream flush failed: %s", err)
|
||||
}
|
||||
close(errCh)
|
||||
}()
|
||||
|
||||
//if err := car.Write(pipeW); err != nil {
|
||||
if err := gocar.WriteCar(
|
||||
req.Context,
|
||||
node.DAG,
|
||||
[]cid.Cid{c},
|
||||
pipeW,
|
||||
); err != nil {
|
||||
errCh <- err
|
||||
}
|
||||
}()
|
||||
|
||||
if err := res.Emit(pipeR); err != nil {
|
||||
pipeW.Close() // ignore errors if any
|
||||
return err
|
||||
}
|
||||
|
||||
return <-errCh
|
||||
},
|
||||
}
|
||||
|
||||
@ -190,6 +190,7 @@ var rootROSubcommands = map[string]*cmds.Command{
|
||||
"dag": {
|
||||
Subcommands: map[string]*cmds.Command{
|
||||
"get": dag.DagGetCmd,
|
||||
"export": dag.DagExportCmd,
|
||||
"resolve": dag.DagResolveCmd,
|
||||
},
|
||||
},
|
||||
|
||||
1
go.mod
1
go.mod
@ -54,6 +54,7 @@ require (
|
||||
github.com/ipfs/go-unixfs v0.2.4
|
||||
github.com/ipfs/go-verifcid v0.0.1
|
||||
github.com/ipfs/interface-go-ipfs-core v0.2.6
|
||||
github.com/ipld/go-car v0.0.5-0.20200316204026-3e2cf7af0fab
|
||||
github.com/jbenet/go-is-domain v1.0.3
|
||||
github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c
|
||||
github.com/jbenet/go-temp-err-catcher v0.1.0
|
||||
|
||||
10
go.sum
10
go.sum
@ -18,8 +18,10 @@ github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBA
|
||||
github.com/alangpierce/go-forceexport v0.0.0-20160317203124-8f1d6941cd75 h1:3ILjVyslFbc4jl1w5TWuvvslFD/nDfR2H8tVaMVLrEY=
|
||||
github.com/alangpierce/go-forceexport v0.0.0-20160317203124-8f1d6941cd75/go.mod h1:uAXEEpARkRhCZfEvy/y0Jcc888f9tHCc1W7/UeEtreE=
|
||||
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
|
||||
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM=
|
||||
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
|
||||
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
|
||||
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4 h1:Hs82Z41s6SdL1CELW+XaDYmOH4hkBN4/N9og/AsOv7E=
|
||||
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
|
||||
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
|
||||
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
|
||||
@ -190,15 +192,12 @@ github.com/ipfs/go-blockservice v0.0.3/go.mod h1:/NNihwTi6V2Yr6g8wBI+BSwPuURpBRM
|
||||
github.com/ipfs/go-blockservice v0.0.7/go.mod h1:EOfb9k/Y878ZTRY/CH0x5+ATtaipfbRhbvNSdgc/7So=
|
||||
github.com/ipfs/go-blockservice v0.1.0/go.mod h1:hzmMScl1kXHg3M2BjTymbVPjv627N7sYcvYaKbop39M=
|
||||
github.com/ipfs/go-blockservice v0.1.1/go.mod h1:t+411r7psEUhLueM8C7aPA7cxCclv4O3VsUVxt9kz2I=
|
||||
github.com/ipfs/go-blockservice v0.1.2 h1:fqFeeu1EG0lGVrqUo+BVJv7LZV31I4ZsyNthCOMAJRc=
|
||||
github.com/ipfs/go-blockservice v0.1.2/go.mod h1:t+411r7psEUhLueM8C7aPA7cxCclv4O3VsUVxt9kz2I=
|
||||
github.com/ipfs/go-blockservice v0.1.3 h1:9XgsPMwwWJSC9uVr2pMDsW2qFTBSkxpGMhmna8mIjPM=
|
||||
github.com/ipfs/go-blockservice v0.1.3/go.mod h1:OTZhFpkgY48kNzbgyvcexW9cHrpjBYIjSR0KoDOFOLU=
|
||||
github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
|
||||
github.com/ipfs/go-cid v0.0.2/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
|
||||
github.com/ipfs/go-cid v0.0.3 h1:UIAh32wymBpStoe83YCzwVQQ5Oy/H0FdxvUS6DJDzms=
|
||||
github.com/ipfs/go-cid v0.0.3/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
|
||||
github.com/ipfs/go-cid v0.0.4 h1:UlfXKrZx1DjZoBhQHmNHLC1fK1dUJDN20Y28A7s+gJ8=
|
||||
github.com/ipfs/go-cid v0.0.4/go.mod h1:4LLaPOQwmk5z9LBgQnpkivrx8BJjUyGwTXCd5Xfj6+M=
|
||||
github.com/ipfs/go-cid v0.0.5 h1:o0Ix8e/ql7Zb5UVUJEUfjsWCIY8t48++9lR8qi6oiJU=
|
||||
github.com/ipfs/go-cid v0.0.5/go.mod h1:plgt+Y5MnOey4vO4UlUazGqdbEXuFYitED67FexhXog=
|
||||
@ -326,6 +325,7 @@ github.com/ipfs/go-merkledag v0.0.6/go.mod h1:QYPdnlvkOg7GnQRofu9XZimC5ZW5Wi3bKy
|
||||
github.com/ipfs/go-merkledag v0.1.0/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk=
|
||||
github.com/ipfs/go-merkledag v0.2.3 h1:aMdkK9G1hEeNvn3VXfiEMLY0iJnbiQQUHnM0HFJREsE=
|
||||
github.com/ipfs/go-merkledag v0.2.3/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk=
|
||||
github.com/ipfs/go-merkledag v0.2.4/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk=
|
||||
github.com/ipfs/go-merkledag v0.3.0 h1:1bXv/ZRPZLVdij/a33CkXMVdxUdred9sz4xyph+0ls0=
|
||||
github.com/ipfs/go-merkledag v0.3.0/go.mod h1:4pymaZLhSLNVuiCITYrpViD6vmfZ/Ws4n/L9tfNv3S4=
|
||||
github.com/ipfs/go-merkledag v0.3.1 h1:3UqWINBEr3/N+r6OwgFXAddDP/8zpQX/8J7IGVOCqRQ=
|
||||
@ -354,6 +354,8 @@ github.com/ipfs/go-verifcid v0.0.1 h1:m2HI7zIuR5TFyQ1b79Da5N9dnnCP1vcu2QqawmWlK2
|
||||
github.com/ipfs/go-verifcid v0.0.1/go.mod h1:5Hrva5KBeIog4A+UpqlaIU+DEstipcJYQQZc0g37pY0=
|
||||
github.com/ipfs/interface-go-ipfs-core v0.2.6 h1:4eeGPJUDWblEurSzpAtL2znDG1xqAoX2aNNIoOFwyuc=
|
||||
github.com/ipfs/interface-go-ipfs-core v0.2.6/go.mod h1:Tihp8zxGpUeE3Tokr94L6zWZZdkRQvG5TL6i9MuNE+s=
|
||||
github.com/ipld/go-car v0.0.5-0.20200316204026-3e2cf7af0fab h1:+3Y6Jb3IBmG3t6e3r6TItnuciOaMOuGW7QIVEUa5vy4=
|
||||
github.com/ipld/go-car v0.0.5-0.20200316204026-3e2cf7af0fab/go.mod h1:yR5AsJ38xTwwgwGpbh60ICtdLPp5lGfuH28PAAzaEhM=
|
||||
github.com/ipld/go-ipld-prime v0.0.2-0.20191108012745-28a82f04c785 h1:fASnkvtR+SmB2y453RxmDD3Uvd4LonVUgFGk9JoDaZs=
|
||||
github.com/ipld/go-ipld-prime v0.0.2-0.20191108012745-28a82f04c785/go.mod h1:bDDSvVz7vaK12FNvMeRYnpRFkSUPNQOiCYQezMD/P3w=
|
||||
github.com/ipld/go-ipld-prime-proto v0.0.0-20191113031812-e32bd156a1e5 h1:lSip43rAdyGA+yRQuy6ju0ucZkWpYc1F2CTQtZTVW/4=
|
||||
@ -554,7 +556,6 @@ github.com/libp2p/go-libp2p-peerstore v0.2.0 h1:XcgJhI8WyUOCbHyRLNEX5542YNj8hnLS
|
||||
github.com/libp2p/go-libp2p-peerstore v0.2.0/go.mod h1:N2l3eVIeAitSg3Pi2ipSrJYnqhVnMNQZo9nkSCuAbnQ=
|
||||
github.com/libp2p/go-libp2p-peerstore v0.2.1 h1:u+gOfsKgu73ZkGWhvckRm03z9C+iS9TrLqpANweELGs=
|
||||
github.com/libp2p/go-libp2p-peerstore v0.2.1/go.mod h1:NQxhNjWxf1d4w6PihR8btWIRjwRLBr4TYKfNgrUkOPA=
|
||||
github.com/libp2p/go-libp2p-peerstore v0.2.2 h1:iqc/m03jHn5doXN3+kS6JKvqQRHEltiXljQB85iVHWE=
|
||||
github.com/libp2p/go-libp2p-peerstore v0.2.2/go.mod h1:NQxhNjWxf1d4w6PihR8btWIRjwRLBr4TYKfNgrUkOPA=
|
||||
github.com/libp2p/go-libp2p-peerstore v0.2.3 h1:MofRq2l3c15vQpEygTetV+zRRrncz+ktiXW7H2EKoEQ=
|
||||
github.com/libp2p/go-libp2p-peerstore v0.2.3/go.mod h1:K8ljLdFn590GMttg/luh4caB/3g0vKuY01psze0upRw=
|
||||
@ -902,6 +903,7 @@ github.com/texttheater/golang-levenshtein v0.0.0-20180516184445-d188e65d659e/go.
|
||||
github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c h1:u6SKchux2yDvFQnDHS3lPnIRmfVJ5Sxy3ao2SIdysLQ=
|
||||
github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM=
|
||||
github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
|
||||
github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
|
||||
github.com/vishvananda/netlink v1.1.0 h1:1iyaYNBLmP6L0220aDnYQpo1QEV4t4hJ+xEEhhJH8j0=
|
||||
github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE=
|
||||
github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df h1:OviZH7qLw/7ZovXvuNyL3XQl8UFofeikI1NW1Gypu7k=
|
||||
|
||||
25
test/sharness/t0054-dag-car-import-export.sh
Executable file
25
test/sharness/t0054-dag-car-import-export.sh
Executable file
@ -0,0 +1,25 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
|
||||
test_description="Test car file import/export functionality"
|
||||
|
||||
. lib/test-lib.sh
|
||||
|
||||
test_init_ipfs
|
||||
|
||||
|
||||
echo "Error: merkledag: not found (currently offline, perhaps retry after attaching to the network)" > offline_fetch_error_expected
|
||||
|
||||
test_expect_success "basic offline export of nonexistent cid" '
|
||||
! ipfs dag export QmYwAPJXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX 2> offline_fetch_error_actual
|
||||
'
|
||||
|
||||
test_expect_success "correct error" '
|
||||
test_cmp offline_fetch_error_expected offline_fetch_error_actual
|
||||
'
|
||||
|
||||
test_expect_success "basic offline export of 'getting started' dag" '
|
||||
ipfs dag export QmS4ustL54uo8FzR9455qaxZwuMiUhyvMcX9Ba8nUH4uVv >/dev/null
|
||||
'
|
||||
|
||||
test_done
|
||||
Reference in New Issue
Block a user