Merge pull request #2795 from ipfs/fix/mfs-caching

rework add-mfs to not use caching

core/commands/add.go

@@ -10,6 +10,8 @@ import (
 	cmds "github.com/ipfs/go-ipfs/commands"
 	files "github.com/ipfs/go-ipfs/commands/files"
 	core "github.com/ipfs/go-ipfs/core"
+	dagtest "github.com/ipfs/go-ipfs/merkledag/test"
+	mfs "github.com/ipfs/go-ipfs/mfs"
 	u "gx/ipfs/QmZNVWh8LLjAavuQ2JXuFmuYH3C11xo988vSgp7UQrTRj1/go-ipfs-util"
 )
 
@@ -156,6 +158,17 @@ You can now refer to the added file in a gateway, like so:
 		fileAdder.Pin = dopin
 		fileAdder.Silent = silent
 
+		if hash {
+			md := dagtest.Mock()
+			mr, err := mfs.NewRoot(req.Context(), md, coreunix.NewDirNode(), nil)
+			if err != nil {
+				res.SetError(err, cmds.ErrNormal)
+				return
+			}
+
+			fileAdder.SetMfsRoot(mr)
+		}
+
 		addAllAndPin := func(f files.File) error {
 			// Iterate over each top-level file and add individually. Otherwise the
 			// single files.File f is treated as a directory, affecting hidden file

core/coreunix/add.go

 package coreunix
 
 import (
-	"bytes"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -68,7 +67,7 @@ type AddedObject struct {
 }
 
 func NewAdder(ctx context.Context, p pin.Pinner, bs bstore.GCBlockstore, ds dag.DAGService) (*Adder, error) {
-	mr, err := mfs.NewRoot(ctx, ds, newDirNode(), nil)
+	mr, err := mfs.NewRoot(ctx, ds, NewDirNode(), nil)
 	if err != nil {
 		return nil, err
 	}
@@ -109,6 +108,10 @@ type Adder struct {
 	tempRoot   key.Key
 }
 
+func (adder *Adder) SetMfsRoot(r *mfs.Root) {
+	adder.mr = r
+}
+
 // Perform the actual add & pin locally, outputting results to reader
 func (adder Adder) add(reader io.Reader) (*dag.Node, error) {
 	chnk, err := chunk.FromString(reader, adder.Chunker)
@@ -214,34 +217,32 @@ func (adder *Adder) Finalize() (*dag.Node, error) {
 	return root.GetNode()
 }
 
-func (adder *Adder) outputDirs(path string, fs mfs.FSNode) error {
-	nd, err := fs.GetNode()
-	if err != nil {
-		return err
-	}
-
-	if !bytes.Equal(nd.Data, folderData) || fs.Type() != mfs.TDir {
+func (adder *Adder) outputDirs(path string, fsn mfs.FSNode) error {
+	switch fsn := fsn.(type) {
+	case *mfs.File:
 		return nil
-	}
-
-	dir, ok := fs.(*mfs.Directory)
-	if !ok {
-		return fmt.Errorf("received FSNode of type TDir that was not a Directory")
-	}
-
-	for _, name := range dir.ListNames() {
-		child, err := dir.Child(name)
-		if err != nil {
-			return err
+	case *mfs.Directory:
+		for _, name := range fsn.ListNames() {
+			child, err := fsn.Child(name)
+			if err != nil {
+				return err
+			}
+
+			childpath := gopath.Join(path, name)
+			err = adder.outputDirs(childpath, child)
+			if err != nil {
+				return err
+			}
 		}
-
-		err = adder.outputDirs(gopath.Join(path, name), child)
+		nd, err := fsn.GetNode()
 		if err != nil {
 			return err
 		}
-	}
 
-	return outputDagnode(adder.Out, path, nd)
+		return outputDagnode(adder.Out, path, nd)
+	default:
+		return fmt.Errorf("unrecognized fsn type: %#v", fsn)
+	}
 }
 
 // Add builds a merkledag from the a reader, pinning all objects to the local
@@ -488,7 +489,7 @@ func NewMemoryDagService() dag.DAGService {
 }
 
 // TODO: generalize this to more than unix-fs nodes.
-func newDirNode() *dag.Node {
+func NewDirNode() *dag.Node {
 	return &dag.Node{Data: unixfs.FolderPBData()}
 }
 

mfs/dir.go

@@ -131,7 +131,7 @@ func (d *Directory) cacheNode(name string, nd *dag.Node) (FSNode, error) {
 		ndir := NewDirectory(d.ctx, name, nd, d, d.dserv)
 		d.childDirs[name] = ndir
 		return ndir, nil
-	case ufspb.Data_File, ufspb.Data_Raw:
+	case ufspb.Data_File, ufspb.Data_Raw, ufspb.Data_Symlink:
 		nfi, err := NewFile(name, nd, d, d.dserv)
 		if err != nil {
 			return nil, err
@@ -338,18 +338,6 @@ func (d *Directory) AddChild(name string, nd *dag.Node) error {
 	}
 
 	d.modTime = time.Now()
-
-	if len(nd.Links) == 0 {
-		nfi, err := NewFile(name, nd, d, d.dserv)
-		if err != nil {
-			return err
-		}
-		d.files[name] = nfi
-	} else {
-		ndir := NewDirectory(d.ctx, name, nd, d, d.dserv)
-		d.childDirs[name] = ndir
-	}
-
 	return nil
 }
 

mfs/file.go

@@ -45,6 +45,20 @@ func (fi *File) Open(flags int, sync bool) (FileDescriptor, error) {
 	node := fi.node
 	fi.nodelk.Unlock()
 
+	fsn, err := ft.FSNodeFromBytes(node.Data)
+	if err != nil {
+		return nil, err
+	}
+
+	switch fsn.Type {
+	default:
+		return nil, fmt.Errorf("unsupported fsnode type for 'file'")
+	case ft.TSymlink:
+		return nil, fmt.Errorf("symlinks not yet supported")
+	case ft.TFile, ft.TRaw:
+		// OK case
+	}
+
 	switch flags {
 	case OpenReadOnly:
 		fi.desclock.RLock()

test/sharness/t0250-files-api.sh

@@ -437,6 +437,29 @@ test_files_api() {
 	test_expect_success "child dir looks right" '
 		verify_dir_contents /
 	'
+
+	# test for https://github.com/ipfs/go-ipfs/issues/2654
+	test_expect_success "create and remove dir" '
+		ipfs files mkdir /test_dir &&
+		ipfs files rm -r "/test_dir"
+	'
+
+	test_expect_success "create test file" '
+		echo "content" | ipfs files write -e "/test_file"
+	'
+
+	test_expect_success "copy test file onto test dir" '
+		ipfs files cp "/test_file" "/test_dir"
+	'
+
+	test_expect_success "test /test_dir" '
+		ipfs files stat "/test_dir" | grep -q "^Type: file"
+	'
+
+	test_expect_success "clean up /test_dir and /test_file" '
+		ipfs files rm -r /test_dir &&
+		ipfs files rm -r /test_file
+	'
 }
 
 # test offline and online

unixfs/format.go

@@ -15,6 +15,7 @@ const (
 	TFile      = pb.Data_File
 	TDirectory = pb.Data_Directory
 	TMetadata  = pb.Data_Metadata
+	TSymlink   = pb.Data_Symlink
 )
 
 var ErrMalformedFileFormat = errors.New("malformed data in file format")