From 234ad1811d9451bb154b4d7889018f802c779c1a Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Fri, 23 Mar 2018 15:56:57 -0700 Subject: [PATCH 1/5] make the tar writer handle sharded ipfs directories makes ipfs get work on sharded directories fixes #4871 License: MIT Signed-off-by: Steven Allen --- unixfs/archive/tar/writer.go | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/unixfs/archive/tar/writer.go b/unixfs/archive/tar/writer.go index 4503f5593..7e6f13894 100644 --- a/unixfs/archive/tar/writer.go +++ b/unixfs/archive/tar/writer.go @@ -39,23 +39,25 @@ func NewWriter(ctx context.Context, dag ipld.DAGService, archive bool, compressi } func (w *Writer) writeDir(nd *mdag.ProtoNode, fpath string) error { + dir, err := uio.NewDirectoryFromNode(w.Dag, nd) + if err != nil { + return err + } if err := writeDirHeader(w.TarW, fpath); err != nil { return err } - for i, ng := range ipld.GetDAG(w.ctx, w.Dag, nd) { - child, err := ng.Get(w.ctx) + return dir.ForEachLink(w.ctx, func(l *ipld.Link) error { + child, err := w.Dag.Get(w.ctx, l.Cid) if err != nil { return err } - - npath := path.Join(fpath, nd.Links()[i].Name) + npath := path.Join(fpath, l.Name) if err := w.WriteNode(child, npath); err != nil { return err } - } - - return nil + return nil + }) } func (w *Writer) writeFile(nd *mdag.ProtoNode, pb *upb.Data, fpath string) error { @@ -83,7 +85,7 @@ func (w *Writer) WriteNode(nd ipld.Node, fpath string) error { switch pb.GetType() { case upb.Data_Metadata: fallthrough - case upb.Data_Directory: + case upb.Data_Directory, upb.Data_HAMTShard: return w.writeDir(nd, fpath) case upb.Data_Raw: fallthrough From dd94858981c736b40e05bb1ac174ab50f256ef6f Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Fri, 23 Mar 2018 15:59:21 -0700 Subject: [PATCH 2/5] fix error when encountering a sharded directory when expecting a file License: MIT Signed-off-by: Steven Allen --- unixfs/io/pbdagreader.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unixfs/io/pbdagreader.go b/unixfs/io/pbdagreader.go index 5be60bd8e..ac08fade8 100644 --- a/unixfs/io/pbdagreader.go +++ b/unixfs/io/pbdagreader.go @@ -112,7 +112,7 @@ func (dr *PBDagReader) precalcNextBuf(ctx context.Context) error { } switch pb.GetType() { - case ftpb.Data_Directory: + case ftpb.Data_Directory, ftpb.Data_HAMTShard: // A directory should not exist within a file return ft.ErrInvalidDirLocation case ftpb.Data_File: From 6895fc653fb5f74efd2e7b846213aa122638d0fc Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Fri, 23 Mar 2018 16:03:02 -0700 Subject: [PATCH 3/5] return a nice error when listing sharded directories Unfortunately, fixing this would require breaking the API. License: MIT Signed-off-by: Steven Allen --- core/commands/unixfs/ls.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/core/commands/unixfs/ls.go b/core/commands/unixfs/ls.go index 9cf05f610..0c989a2f8 100644 --- a/core/commands/unixfs/ls.go +++ b/core/commands/unixfs/ls.go @@ -137,6 +137,10 @@ possible, please use 'ipfs ls' instead. switch t { case unixfspb.Data_File: break + case unixfspb.Data_HAMTShard: + // We need a streaming ls API for this. + res.SetError(fmt.Errorf("cannot list large directories yet"), cmdkit.ErrNormal) + return case unixfspb.Data_Directory: links := make([]LsLink, len(merkleNode.Links())) output.Objects[hash].Links = links From 0dc984290cb2f839f94254f4c3adb7818c85e4be Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Fri, 23 Mar 2018 16:24:57 -0700 Subject: [PATCH 4/5] test ipfs get on sharded directory License: MIT Signed-off-by: Steven Allen --- test/sharness/{t0260-sharding-flag.sh => t0260-sharding.sh} | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) rename test/sharness/{t0260-sharding-flag.sh => t0260-sharding.sh} (91%) diff --git a/test/sharness/t0260-sharding-flag.sh b/test/sharness/t0260-sharding.sh similarity index 91% rename from test/sharness/t0260-sharding-flag.sh rename to test/sharness/t0260-sharding.sh index 18dd6a05e..2556272e2 100755 --- a/test/sharness/t0260-sharding-flag.sh +++ b/test/sharness/t0260-sharding.sh @@ -4,7 +4,7 @@ # MIT Licensed; see the LICENSE file in this repository. # -test_description="Test global enable sharding flag" +test_description="Test directory sharding" . lib/test-lib.sh @@ -23,6 +23,10 @@ test_add_large_dir() { echo "$exphash" > sharddir_exp && test_cmp sharddir_exp sharddir_out ' + test_expect_success "ipfs get on very large directory succeeds" ' + ipfs get -o testdata-out "$exphash" && + test_cmp testdata testdata-out + ' } test_init_ipfs From 71a0a5c837d6baa1a97bad3776175a19166de7ad Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Fri, 23 Mar 2018 18:02:02 -0700 Subject: [PATCH 5/5] appease go lint License: MIT Signed-off-by: Steven Allen --- unixfs/archive/tar/writer.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/unixfs/archive/tar/writer.go b/unixfs/archive/tar/writer.go index 7e6f13894..04f5bc4cc 100644 --- a/unixfs/archive/tar/writer.go +++ b/unixfs/archive/tar/writer.go @@ -53,10 +53,7 @@ func (w *Writer) writeDir(nd *mdag.ProtoNode, fpath string) error { return err } npath := path.Join(fpath, l.Name) - if err := w.WriteNode(child, npath); err != nil { - return err - } - return nil + return w.WriteNode(child, npath) }) }