From d993bc04d6b21bd76a5a9f0a7924b48e0e09b275 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Wed, 12 Aug 2015 12:17:52 -0700 Subject: [PATCH] implement symlinks in unixfs, first draft License: MIT Signed-off-by: Jeromy --- commands/files/linkfile.go | 50 ++++++++++++++++++++++++++++++++ commands/files/multipartfile.go | 12 ++++++++ commands/files/serialfile.go | 15 ++++++++++ commands/http/multifilereader.go | 21 +++++++++----- core/commands/add.go | 18 ++++++++++++ fuse/readonly/readonly_unix.go | 19 ++++++++++++ unixfs/format.go | 14 +++++++++ unixfs/io/dagreader.go | 6 ++++ unixfs/pb/unixfs.pb.go | 5 +++- unixfs/pb/unixfs.proto | 1 + 10 files changed, 152 insertions(+), 9 deletions(-) create mode 100644 commands/files/linkfile.go diff --git a/commands/files/linkfile.go b/commands/files/linkfile.go new file mode 100644 index 000000000..18466f4bd --- /dev/null +++ b/commands/files/linkfile.go @@ -0,0 +1,50 @@ +package files + +import ( + "io" + "os" + "strings" +) + +type Symlink struct { + name string + path string + Target string + stat os.FileInfo + + reader io.Reader +} + +func NewLinkFile(name, path, target string, stat os.FileInfo) File { + return &Symlink{ + name: name, + path: path, + Target: target, + stat: stat, + reader: strings.NewReader(target), + } +} + +func (lf *Symlink) IsDirectory() bool { + return false +} + +func (lf *Symlink) NextFile() (File, error) { + return nil, io.EOF +} + +func (f *Symlink) FileName() string { + return f.name +} + +func (f *Symlink) Close() error { + return nil +} + +func (f *Symlink) FullPath() string { + return f.path +} + +func (f *Symlink) Read(b []byte) (int, error) { + return f.reader.Read(b) +} diff --git a/commands/files/multipartfile.go b/commands/files/multipartfile.go index b68224cbd..b43cb8624 100644 --- a/commands/files/multipartfile.go +++ b/commands/files/multipartfile.go @@ -1,6 +1,7 @@ package files import ( + "io/ioutil" "mime" "mime/multipart" "net/http" @@ -30,6 +31,17 @@ func NewFileFromPart(part *multipart.Part) (File, error) { } contentType := part.Header.Get(contentTypeHeader) + if contentType == "symlink" { + out, err := ioutil.ReadAll(part) + if err != nil { + return nil, err + } + + return &Symlink{ + Target: string(out), + name: f.FileName(), + }, nil + } var params map[string]string var err error diff --git a/commands/files/serialfile.go b/commands/files/serialfile.go index 4b15e4434..34b831e6f 100644 --- a/commands/files/serialfile.go +++ b/commands/files/serialfile.go @@ -84,10 +84,25 @@ func (f *serialFile) NextFile() (File, error) { // open the next file fileName := fp.Join(f.name, stat.Name()) filePath := fp.Join(f.path, stat.Name()) + st, err := os.Lstat(filePath) + if err != nil { + return nil, err + } + + if st.Mode()&os.ModeSymlink != 0 { + f.current = nil + target, err := os.Readlink(filePath) + if err != nil { + return nil, err + } + return NewLinkFile(fileName, filePath, target, st), nil + } + file, err := os.Open(filePath) if err != nil { return nil, err } + f.current = file // recursively call the constructor on the next file diff --git a/commands/http/multifilereader.go b/commands/http/multifilereader.go index 711117896..83d59dc9e 100644 --- a/commands/http/multifilereader.go +++ b/commands/http/multifilereader.go @@ -64,13 +64,23 @@ func (mfr *MultiFileReader) Read(buf []byte) (written int, err error) { // handle starting a new file part if !mfr.closed { - if file.IsDirectory() { + + var contentType string + if s, ok := file.(*files.Symlink); ok { + mfr.currentFile = s + + // TODO(why): this is a hack. pick a real contentType + contentType = "symlink" + } else if file.IsDirectory() { // if file is a directory, create a multifilereader from it // (using 'multipart/mixed') - mfr.currentFile = NewMultiFileReader(file, false) + nmfr := NewMultiFileReader(file, false) + mfr.currentFile = nmfr + contentType = fmt.Sprintf("multipart/mixed; boundary=%s", nmfr.Boundary()) } else { // otherwise, use the file as a reader to read its contents mfr.currentFile = file + contentType = "application/octet-stream" } // write the boundary and headers @@ -83,12 +93,7 @@ func (mfr *MultiFileReader) Read(buf []byte) (written int, err error) { header.Set("Content-Disposition", fmt.Sprintf("file; filename=\"%s\"", filename)) } - if file.IsDirectory() { - boundary := mfr.currentFile.(*MultiFileReader).Boundary() - header.Set("Content-Type", fmt.Sprintf("multipart/mixed; boundary=%s", boundary)) - } else { - header.Set("Content-Type", "application/octet-stream") - } + header.Set("Content-Type", contentType) _, err := mfr.mpWriter.CreatePart(header) if err != nil { diff --git a/core/commands/add.go b/core/commands/add.go index 458336e43..ecc9ad42d 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -143,6 +143,7 @@ remains to be implemented. return nil // done } + log.Errorf("FILE: %#v", file) if _, err := fileAdder.addFile(file); err != nil { return err } @@ -359,6 +360,23 @@ func (params *adder) addFile(file files.File) (*dag.Node, error) { return params.addDir(file) } + if s, ok := file.(*files.Symlink); ok { + log.Error("SYMLINK: ", s) + log.Error(s.Target) + log.Error(s.FileName()) + dagnode := &dag.Node{ + Data: ft.SymlinkData(s.Target), + } + + _, err := params.node.DAG.Add(dagnode) + if err != nil { + return nil, err + } + + err = params.addNode(dagnode, s.FileName()) + return dagnode, err + } + // if the progress flag was specified, wrap the file so that we can send // progress updates to the client (over the output channel) var reader io.Reader = file diff --git a/fuse/readonly/readonly_unix.go b/fuse/readonly/readonly_unix.go index 290618791..d032f7bf1 100644 --- a/fuse/readonly/readonly_unix.go +++ b/fuse/readonly/readonly_unix.go @@ -7,6 +7,8 @@ import ( "fmt" "io" "os" + "syscall" + "time" fuse "github.com/ipfs/go-ipfs/Godeps/_workspace/src/bazil.org/fuse" fs "github.com/ipfs/go-ipfs/Godeps/_workspace/src/bazil.org/fuse/fs" @@ -58,6 +60,8 @@ func (s *Root) Lookup(ctx context.Context, name string) (fs.Node, error) { return nil, fuse.ENOENT } + log.Error("RESOLVE: ", name) + ctx, _ = context.WithTimeout(ctx, time.Second/2) nd, err := s.Ipfs.Resolver.ResolvePath(ctx, path.Path(name)) if err != nil { // todo: make this error more versatile. @@ -118,6 +122,13 @@ func (s *Node) Attr(ctx context.Context, a *fuse.Attr) error { Uid: uint32(os.Getuid()), Gid: uint32(os.Getgid()), } + case ftpb.Data_Symlink: + *a = fuse.Attr{ + Mode: 0777 | os.ModeSymlink, + Size: uint64(len(s.cached.GetData())), + Uid: uint32(os.Getuid()), + Gid: uint32(os.Getgid()), + } default: return fmt.Errorf("Invalid data type - %s", s.cached.GetType()) @@ -155,6 +166,13 @@ func (s *Node) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) { return nil, fuse.ENOENT } +func (s *Node) Readlink(ctx context.Context, req *fuse.ReadlinkRequest) (string, error) { + if s.cached.GetType() != ftpb.Data_Symlink { + return "", fuse.Errno(syscall.EINVAL) + } + return string(s.cached.GetData()), nil +} + func (s *Node) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse.ReadResponse) error { k, err := s.Nd.Key() @@ -204,6 +222,7 @@ type roNode interface { fs.HandleReader fs.Node fs.NodeStringLookuper + fs.NodeReadlinker } var _ roNode = (*Node)(nil) diff --git a/unixfs/format.go b/unixfs/format.go index b8c0abeb1..c1f82a485 100644 --- a/unixfs/format.go +++ b/unixfs/format.go @@ -77,6 +77,20 @@ func WrapData(b []byte) []byte { return out } +func SymlinkData(path string) []byte { + pbdata := new(pb.Data) + typ := pb.Data_Symlink + pbdata.Data = []byte(path) + pbdata.Type = &typ + + out, err := proto.Marshal(pbdata) + if err != nil { + panic(err) + } + + return out +} + func UnwrapData(data []byte) ([]byte, error) { pbdata := new(pb.Data) err := proto.Unmarshal(data, pbdata) diff --git a/unixfs/io/dagreader.go b/unixfs/io/dagreader.go index 1426f10cc..646a69a40 100644 --- a/unixfs/io/dagreader.go +++ b/unixfs/io/dagreader.go @@ -17,6 +17,8 @@ import ( var ErrIsDir = errors.New("this dag node is a directory") +var ErrCantReadSymlinks = errors.New("cannot currently read symlinks") + // DagReader provides a way to easily read the data contained in a dag. type DagReader struct { serv mdag.DAGService @@ -79,6 +81,8 @@ func NewDagReader(ctx context.Context, n *mdag.Node, serv mdag.DAGService) (*Dag return nil, err } return NewDagReader(ctx, child, serv) + case ftpb.Data_Symlink: + return nil, ErrCantReadSymlinks default: return nil, ft.ErrUnrecognizedType } @@ -130,6 +134,8 @@ func (dr *DagReader) precalcNextBuf(ctx context.Context) error { return nil case ftpb.Data_Metadata: return errors.New("Shouldnt have had metadata object inside file") + case ftpb.Data_Symlink: + return errors.New("shouldnt have had symlink inside file") default: return ft.ErrUnrecognizedType } diff --git a/unixfs/pb/unixfs.pb.go b/unixfs/pb/unixfs.pb.go index c11ffd4d0..074a32998 100644 --- a/unixfs/pb/unixfs.pb.go +++ b/unixfs/pb/unixfs.pb.go @@ -14,7 +14,7 @@ It has these top-level messages: */ package unixfs_pb -import proto "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/gogo/protobuf/proto" +import proto "github.com/gogo/protobuf/proto" import math "math" // Reference imports to suppress errors if they are not otherwise used. @@ -28,6 +28,7 @@ const ( Data_Directory Data_DataType = 1 Data_File Data_DataType = 2 Data_Metadata Data_DataType = 3 + Data_Symlink Data_DataType = 4 ) var Data_DataType_name = map[int32]string{ @@ -35,12 +36,14 @@ var Data_DataType_name = map[int32]string{ 1: "Directory", 2: "File", 3: "Metadata", + 4: "Symlink", } var Data_DataType_value = map[string]int32{ "Raw": 0, "Directory": 1, "File": 2, "Metadata": 3, + "Symlink": 4, } func (x Data_DataType) Enum() *Data_DataType { diff --git a/unixfs/pb/unixfs.proto b/unixfs/pb/unixfs.proto index 1450809e4..4a52c3af5 100644 --- a/unixfs/pb/unixfs.proto +++ b/unixfs/pb/unixfs.proto @@ -6,6 +6,7 @@ message Data { Directory = 1; File = 2; Metadata = 3; + Symlink = 4; } required DataType Type = 1;