Merge pull request #1627 from ipfs/feat/symlinks

implement symlinks in unixfs, first draft

commands/cli/parse.go

@@ -351,13 +351,22 @@ func appendFile(args []files.File, inputs []string, argDef *cmds.Argument, recur
 		}
 		fpath = cwd
 	}
-
-	file, err := os.Open(fpath)
+	stat, err := os.Lstat(fpath)
 	if err != nil {
 		return nil, nil, err
 	}
 
-	stat, err := file.Stat()
+	if stat.Mode()&os.ModeSymlink != 0 {
+		target, err := os.Readlink(fpath)
+		if err != nil {
+			return nil, nil, err
+		}
+
+		arg := files.NewLinkFile("", fpath, target, stat)
+		return append(args, arg), inputs[1:], nil
+	}
+
+	file, err := os.Open(fpath)
 	if err != nil {
 		return nil, nil, err
 	}

commands/files/linkfile.go

+package files
+
+import (
+	"io"
+	"os"
+	"strings"
+)
+
+type Symlink struct {
+	name   string
+	path   string
+	Target string
+	stat   os.FileInfo
+
+	reader io.Reader
+}
+
+func NewLinkFile(name, path, target string, stat os.FileInfo) File {
+	return &Symlink{
+		name:   name,
+		path:   path,
+		Target: target,
+		stat:   stat,
+		reader: strings.NewReader(target),
+	}
+}
+
+func (lf *Symlink) IsDirectory() bool {
+	return false
+}
+
+func (lf *Symlink) NextFile() (File, error) {
+	return nil, io.EOF
+}
+
+func (f *Symlink) FileName() string {
+	return f.name
+}
+
+func (f *Symlink) Close() error {
+	return nil
+}
+
+func (f *Symlink) FullPath() string {
+	return f.path
+}
+
+func (f *Symlink) Read(b []byte) (int, error) {
+	return f.reader.Read(b)
+}

commands/files/multipartfile.go

 package files
 
 import (
+	"io/ioutil"
 	"mime"
 	"mime/multipart"
 	"net/http"
@@ -11,6 +12,8 @@ const (
 	multipartFormdataType = "multipart/form-data"
 	multipartMixedType    = "multipart/mixed"
 
+	applicationSymlink = "application/symlink"
+
 	contentTypeHeader = "Content-Type"
 )
 
@@ -30,6 +33,17 @@ func NewFileFromPart(part *multipart.Part) (File, error) {
 	}
 
 	contentType := part.Header.Get(contentTypeHeader)
+	if contentType == applicationSymlink {
+		out, err := ioutil.ReadAll(part)
+		if err != nil {
+			return nil, err
+		}
+
+		return &Symlink{
+			Target: string(out),
+			name:   f.FileName(),
+		}, nil
+	}
 
 	var params map[string]string
 	var err error

commands/files/serialfile.go

@@ -84,10 +84,25 @@ func (f *serialFile) NextFile() (File, error) {
 	// open the next file
 	fileName := fp.Join(f.name, stat.Name())
 	filePath := fp.Join(f.path, stat.Name())
+	st, err := os.Lstat(filePath)
+	if err != nil {
+		return nil, err
+	}
+
+	if st.Mode()&os.ModeSymlink != 0 {
+		f.current = nil
+		target, err := os.Readlink(filePath)
+		if err != nil {
+			return nil, err
+		}
+		return NewLinkFile(fileName, filePath, target, st), nil
+	}
+
 	file, err := os.Open(filePath)
 	if err != nil {
 		return nil, err
 	}
+
 	f.current = file
 
 	// recursively call the constructor on the next file

commands/http/multifilereader.go

@@ -64,13 +64,22 @@ func (mfr *MultiFileReader) Read(buf []byte) (written int, err error) {
 
 		// handle starting a new file part
 		if !mfr.closed {
-			if file.IsDirectory() {
+
+			var contentType string
+			if s, ok := file.(*files.Symlink); ok {
+				mfr.currentFile = s
+
+				contentType = "application/symlink"
+			} else if file.IsDirectory() {
 				// if file is a directory, create a multifilereader from it
 				// (using 'multipart/mixed')
-				mfr.currentFile = NewMultiFileReader(file, false)
+				nmfr := NewMultiFileReader(file, false)
+				mfr.currentFile = nmfr
+				contentType = fmt.Sprintf("multipart/mixed; boundary=%s", nmfr.Boundary())
 			} else {
 				// otherwise, use the file as a reader to read its contents
 				mfr.currentFile = file
+				contentType = "application/octet-stream"
 			}
 
 			// write the boundary and headers
@@ -83,12 +92,7 @@ func (mfr *MultiFileReader) Read(buf []byte) (written int, err error) {
 				header.Set("Content-Disposition", fmt.Sprintf("file; filename=\"%s\"", filename))
 			}
 
-			if file.IsDirectory() {
-				boundary := mfr.currentFile.(*MultiFileReader).Boundary()
-				header.Set("Content-Type", fmt.Sprintf("multipart/mixed; boundary=%s", boundary))
-			} else {
-				header.Set("Content-Type", "application/octet-stream")
-			}
+			header.Set("Content-Type", contentType)
 
 			_, err := mfr.mpWriter.CreatePart(header)
 			if err != nil {

core/commands/add.go

@@ -359,6 +359,22 @@ func (params *adder) addFile(file files.File) (*dag.Node, error) {
 		return params.addDir(file)
 	}
 
+	if s, ok := file.(*files.Symlink); ok {
+		sdata, err := ft.SymlinkData(s.Target)
+		if err != nil {
+			return nil, err
+		}
+
+		dagnode := &dag.Node{Data: sdata}
+		_, err = params.node.DAG.Add(dagnode)
+		if err != nil {
+			return nil, err
+		}
+
+		err = params.addNode(dagnode, s.FileName())
+		return dagnode, err
+	}
+
 	// if the progress flag was specified, wrap the file so that we can send
 	// progress updates to the client (over the output channel)
 	var reader io.Reader = file

core/commands/unixfs/ls.go

@@ -99,9 +99,6 @@ size is the IPFS link size.
 			}
 
 			switch t {
-			default:
-				res.SetError(fmt.Errorf("unrecognized type: %s", t), cmds.ErrImplementation)
-				return
 			case unixfspb.Data_File:
 				break
 			case unixfspb.Data_Directory:
@@ -131,6 +128,12 @@ size is the IPFS link size.
 					}
 					links[i] = lsLink
 				}
+			case unixfspb.Data_Symlink:
+				res.SetError(fmt.Errorf("cannot list symlinks yet"), cmds.ErrNormal)
+				return
+			default:
+				res.SetError(fmt.Errorf("unrecognized type: %s", t), cmds.ErrImplementation)
+				return
 			}
 		}
 

fuse/readonly/readonly_unix.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"syscall"
 
 	fuse "github.com/ipfs/go-ipfs/Godeps/_workspace/src/bazil.org/fuse"
 	fs "github.com/ipfs/go-ipfs/Godeps/_workspace/src/bazil.org/fuse/fs"
@@ -96,28 +97,27 @@ func (s *Node) Attr(ctx context.Context, a *fuse.Attr) error {
 	}
 	switch s.cached.GetType() {
 	case ftpb.Data_Directory:
-		*a = fuse.Attr{
-			Mode: os.ModeDir | 0555,
-			Uid:  uint32(os.Getuid()),
-			Gid:  uint32(os.Getgid()),
-		}
+		a.Mode = os.ModeDir | 0555
+		a.Uid = uint32(os.Getuid())
+		a.Gid = uint32(os.Getgid())
 	case ftpb.Data_File:
 		size := s.cached.GetFilesize()
-		*a = fuse.Attr{
-			Mode:   0444,
-			Size:   uint64(size),
-			Blocks: uint64(len(s.Nd.Links)),
-			Uid:    uint32(os.Getuid()),
-			Gid:    uint32(os.Getgid()),
-		}
+		a.Mode = 0444
+		a.Size = uint64(size)
+		a.Blocks = uint64(len(s.Nd.Links))
+		a.Uid = uint32(os.Getuid())
+		a.Gid = uint32(os.Getgid())
 	case ftpb.Data_Raw:
-		*a = fuse.Attr{
-			Mode:   0444,
-			Size:   uint64(len(s.cached.GetData())),
-			Blocks: uint64(len(s.Nd.Links)),
-			Uid:    uint32(os.Getuid()),
-			Gid:    uint32(os.Getgid()),
-		}
+		a.Mode = 0444
+		a.Size = uint64(len(s.cached.GetData()))
+		a.Blocks = uint64(len(s.Nd.Links))
+		a.Uid = uint32(os.Getuid())
+		a.Gid = uint32(os.Getgid())
+	case ftpb.Data_Symlink:
+		a.Mode = 0777 | os.ModeSymlink
+		a.Size = uint64(len(s.cached.GetData()))
+		a.Uid = uint32(os.Getuid())
+		a.Gid = uint32(os.Getgid())
 
 	default:
 		return fmt.Errorf("Invalid data type - %s", s.cached.GetType())
@@ -155,6 +155,13 @@ func (s *Node) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) {
 	return nil, fuse.ENOENT
 }
 
+func (s *Node) Readlink(ctx context.Context, req *fuse.ReadlinkRequest) (string, error) {
+	if s.cached.GetType() != ftpb.Data_Symlink {
+		return "", fuse.Errno(syscall.EINVAL)
+	}
+	return string(s.cached.GetData()), nil
+}
+
 func (s *Node) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse.ReadResponse) error {
 
 	k, err := s.Nd.Key()
@@ -204,6 +211,7 @@ type roNode interface {
 	fs.HandleReader
 	fs.Node
 	fs.NodeStringLookuper
+	fs.NodeReadlinker
 }
 
 var _ roNode = (*Node)(nil)

test/sharness/t0044-add-symlink.sh

+#!/bin/bash
+#
+# Copyright (c) 2014 Christian Couder
+# MIT Licensed; see the LICENSE file in this repository.
+#
+
+test_description="Test add -w"
+
+. lib/test-lib.sh
+
+test_expect_success "creating files succeeds" '
+	mkdir -p files/foo &&
+	mkdir -p files/bar &&
+	echo "some text" > files/foo/baz &&
+	ln -s files/foo/baz files/bar/baz &&
+	ln -s files/does/not/exist files/bad
+'
+
+test_add_symlinks() {
+	test_expect_success "ipfs add files succeeds" '
+		ipfs add -q -r files | tail -n 1 > filehash_out
+	'
+
+	test_expect_success "output looks good" '
+		echo QmWdiHKoeSW8G1u7ATCgpx4yMoUhYaJBQGkyPLkS9goYZ8 > filehash_exp &&
+		test_cmp filehash_out filehash_exp
+	'
+
+	test_expect_success "adding a symlink adds the link itself" '
+		ipfs add -q files/bar/baz > goodlink_out
+	'
+
+	test_expect_success "output looks good" '
+		echo "QmdocmZeF7qwPT9Z8SiVhMSyKA2KKoA2J7jToW6z6WBmxR" > goodlink_exp &&
+		test_cmp goodlink_out goodlink_exp
+	'
+
+	test_expect_success "adding a broken symlink works" '
+		ipfs add -q files/bad > badlink_out
+	'
+
+	test_expect_success "output looks good" '
+		echo "QmWYN8SEXCgNT2PSjB6BnxAx6NJQtazWoBkTRH9GRfPFFQ" > badlink_exp &&
+		test_cmp badlink_out badlink_exp
+	'
+}
+
+test_init_ipfs
+
+test_add_symlinks
+
+test_launch_ipfs_daemon
+
+test_add_symlinks
+
+test_kill_ipfs_daemon
+
+test_done

thirdparty/tar/extractor.go

@@ -2,6 +2,7 @@ package tar
 
 import (
 	"archive/tar"
+	"fmt"
 	"io"
 	"os"
 	gopath "path"
@@ -39,15 +40,21 @@ func (te *Extractor) Extract(reader io.Reader) error {
 			break
 		}
 
-		if header.Typeflag == tar.TypeDir {
+		switch header.Typeflag {
+		case tar.TypeDir:
 			if err := te.extractDir(header, i); err != nil {
 				return err
 			}
-			continue
-		}
-
-		if err := te.extractFile(header, tarReader, i, rootExists, rootIsDir); err != nil {
-			return err
+		case tar.TypeReg:
+			if err := te.extractFile(header, tarReader, i, rootExists, rootIsDir); err != nil {
+				return err
+			}
+		case tar.TypeSymlink:
+			if err := te.extractSymlink(header); err != nil {
+				return err
+			}
+		default:
+			return fmt.Errorf("unrecognized tar header type: %d", header.Typeflag)
 		}
 	}
 	return nil
@@ -79,6 +86,10 @@ func (te *Extractor) extractDir(h *tar.Header, depth int) error {
 	return nil
 }
 
+func (te *Extractor) extractSymlink(h *tar.Header) error {
+	return os.Symlink(h.Linkname, te.outputPath(h.Name))
+}
+
 func (te *Extractor) extractFile(h *tar.Header, r *tar.Reader, depth int, rootExists bool, rootIsDir bool) error {
 	path := te.outputPath(h.Name)
 

unixfs/archive/tar/writer.go

@@ -79,6 +79,8 @@ func (w *Writer) WriteNode(nd *mdag.Node, fpath string) error {
 		fallthrough
 	case upb.Data_File:
 		return w.writeFile(nd, pb, fpath)
+	case upb.Data_Symlink:
+		return writeSymlinkHeader(w.TarW, string(pb.GetData()), fpath)
 	default:
 		return ft.ErrUnrecognizedType
 	}
@@ -108,3 +110,12 @@ func writeFileHeader(w *tar.Writer, fpath string, size uint64) error {
 		// TODO: set mode, dates, etc. when added to unixFS
 	})
 }
+
+func writeSymlinkHeader(w *tar.Writer, target, fpath string) error {
+	return w.WriteHeader(&tar.Header{
+		Name:     fpath,
+		Linkname: target,
+		Mode:     0777,
+		Typeflag: tar.TypeSymlink,
+	})
+}

unixfs/format.go

@@ -77,6 +77,20 @@ func WrapData(b []byte) []byte {
 	return out
 }
 
+func SymlinkData(path string) ([]byte, error) {
+	pbdata := new(pb.Data)
+	typ := pb.Data_Symlink
+	pbdata.Data = []byte(path)
+	pbdata.Type = &typ
+
+	out, err := proto.Marshal(pbdata)
+	if err != nil {
+		return nil, err
+	}
+
+	return out, nil
+}
+
 func UnwrapData(data []byte) ([]byte, error) {
 	pbdata := new(pb.Data)
 	err := proto.Unmarshal(data, pbdata)

unixfs/io/dagreader.go

@@ -17,6 +17,8 @@ import (
 
 var ErrIsDir = errors.New("this dag node is a directory")
 
+var ErrCantReadSymlinks = errors.New("cannot currently read symlinks")
+
 // DagReader provides a way to easily read the data contained in a dag.
 type DagReader struct {
 	serv mdag.DAGService
@@ -79,6 +81,8 @@ func NewDagReader(ctx context.Context, n *mdag.Node, serv mdag.DAGService) (*Dag
 			return nil, err
 		}
 		return NewDagReader(ctx, child, serv)
+	case ftpb.Data_Symlink:
+		return nil, ErrCantReadSymlinks
 	default:
 		return nil, ft.ErrUnrecognizedType
 	}
@@ -130,6 +134,8 @@ func (dr *DagReader) precalcNextBuf(ctx context.Context) error {
 		return nil
 	case ftpb.Data_Metadata:
 		return errors.New("Shouldnt have had metadata object inside file")
+	case ftpb.Data_Symlink:
+		return errors.New("shouldnt have had symlink inside file")
 	default:
 		return ft.ErrUnrecognizedType
 	}

unixfs/pb/unixfs.pb.go

@@ -14,7 +14,7 @@ It has these top-level messages:
 */
 package unixfs_pb
 
-import proto "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/gogo/protobuf/proto"
+import proto "github.com/gogo/protobuf/proto"
 import math "math"
 
 // Reference imports to suppress errors if they are not otherwise used.
@@ -28,6 +28,7 @@ const (
 	Data_Directory Data_DataType = 1
 	Data_File      Data_DataType = 2
 	Data_Metadata  Data_DataType = 3
+	Data_Symlink   Data_DataType = 4
 )
 
 var Data_DataType_name = map[int32]string{
@@ -35,12 +36,14 @@ var Data_DataType_name = map[int32]string{
 	1: "Directory",
 	2: "File",
 	3: "Metadata",
+	4: "Symlink",
 }
 var Data_DataType_value = map[string]int32{
 	"Raw":       0,
 	"Directory": 1,
 	"File":      2,
 	"Metadata":  3,
+	"Symlink":   4,
 }
 
 func (x Data_DataType) Enum() *Data_DataType {

unixfs/pb/unixfs.proto

@@ -6,6 +6,7 @@ message Data {
 		Directory = 1;
 		File = 2;
 		Metadata = 3;
+		Symlink = 4;
 	}
 
 	required DataType Type = 1;