diff --git a/core/commands/unixfs/ls.go b/core/commands/unixfs/ls.go index 9cf05f610..0c989a2f8 100644 --- a/core/commands/unixfs/ls.go +++ b/core/commands/unixfs/ls.go @@ -137,6 +137,10 @@ possible, please use 'ipfs ls' instead. switch t { case unixfspb.Data_File: break + case unixfspb.Data_HAMTShard: + // We need a streaming ls API for this. + res.SetError(fmt.Errorf("cannot list large directories yet"), cmdkit.ErrNormal) + return case unixfspb.Data_Directory: links := make([]LsLink, len(merkleNode.Links())) output.Objects[hash].Links = links diff --git a/test/sharness/t0260-sharding-flag.sh b/test/sharness/t0260-sharding.sh similarity index 91% rename from test/sharness/t0260-sharding-flag.sh rename to test/sharness/t0260-sharding.sh index 18dd6a05e..2556272e2 100755 --- a/test/sharness/t0260-sharding-flag.sh +++ b/test/sharness/t0260-sharding.sh @@ -4,7 +4,7 @@ # MIT Licensed; see the LICENSE file in this repository. # -test_description="Test global enable sharding flag" +test_description="Test directory sharding" . lib/test-lib.sh @@ -23,6 +23,10 @@ test_add_large_dir() { echo "$exphash" > sharddir_exp && test_cmp sharddir_exp sharddir_out ' + test_expect_success "ipfs get on very large directory succeeds" ' + ipfs get -o testdata-out "$exphash" && + test_cmp testdata testdata-out + ' } test_init_ipfs diff --git a/unixfs/archive/tar/writer.go b/unixfs/archive/tar/writer.go index 4503f5593..04f5bc4cc 100644 --- a/unixfs/archive/tar/writer.go +++ b/unixfs/archive/tar/writer.go @@ -39,23 +39,22 @@ func NewWriter(ctx context.Context, dag ipld.DAGService, archive bool, compressi } func (w *Writer) writeDir(nd *mdag.ProtoNode, fpath string) error { + dir, err := uio.NewDirectoryFromNode(w.Dag, nd) + if err != nil { + return err + } if err := writeDirHeader(w.TarW, fpath); err != nil { return err } - for i, ng := range ipld.GetDAG(w.ctx, w.Dag, nd) { - child, err := ng.Get(w.ctx) + return dir.ForEachLink(w.ctx, func(l *ipld.Link) error { + child, err := w.Dag.Get(w.ctx, l.Cid) if err != nil { return err } - - npath := path.Join(fpath, nd.Links()[i].Name) - if err := w.WriteNode(child, npath); err != nil { - return err - } - } - - return nil + npath := path.Join(fpath, l.Name) + return w.WriteNode(child, npath) + }) } func (w *Writer) writeFile(nd *mdag.ProtoNode, pb *upb.Data, fpath string) error { @@ -83,7 +82,7 @@ func (w *Writer) WriteNode(nd ipld.Node, fpath string) error { switch pb.GetType() { case upb.Data_Metadata: fallthrough - case upb.Data_Directory: + case upb.Data_Directory, upb.Data_HAMTShard: return w.writeDir(nd, fpath) case upb.Data_Raw: fallthrough diff --git a/unixfs/io/pbdagreader.go b/unixfs/io/pbdagreader.go index 5be60bd8e..ac08fade8 100644 --- a/unixfs/io/pbdagreader.go +++ b/unixfs/io/pbdagreader.go @@ -112,7 +112,7 @@ func (dr *PBDagReader) precalcNextBuf(ctx context.Context) error { } switch pb.GetType() { - case ftpb.Data_Directory: + case ftpb.Data_Directory, ftpb.Data_HAMTShard: // A directory should not exist within a file return ft.ErrInvalidDirLocation case ftpb.Data_File: