fix add/cat of small files

License: MIT
Signed-off-by: Jeromy <why@ipfs.io>

core/commands/files/files.go

@@ -2,6 +2,7 @@ package commands
 
 import (
 	"bytes"
+	"context"
 	"errors"
 	"fmt"
 	"io"
@@ -16,8 +17,8 @@ import (
 	path "github.com/ipfs/go-ipfs/path"
 	ft "github.com/ipfs/go-ipfs/unixfs"
 
-	context "context"
 	logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log"
+	node "gx/ipfs/QmZx42H5khbVQhV5odp66TApShV4XCujYazcvYduZ4TroB/go-ipld-node"
 )
 
 var log = logging.Logger("cmds/files")
@@ -160,7 +161,12 @@ func statNode(ds dag.DAGService, fsn mfs.FSNode) (*Object, error) {
 
 	c := nd.Cid()
 
-	d, err := ft.FromBytes(nd.Data())
+	pbnd, ok := nd.(*dag.ProtoNode)
+	if !ok {
+		return nil, dag.ErrNotProtobuf
+	}
+
+	d, err := ft.FromBytes(pbnd.Data())
 	if err != nil {
 		return nil, err
 	}
@@ -245,7 +251,7 @@ var FilesCpCmd = &cmds.Command{
 	},
 }
 
-func getNodeFromPath(ctx context.Context, node *core.IpfsNode, p string) (*dag.ProtoNode, error) {
+func getNodeFromPath(ctx context.Context, node *core.IpfsNode, p string) (node.Node, error) {
 	switch {
 	case strings.HasPrefix(p, "/ipfs/"):
 		np, err := path.ParsePath(p)

core/corehttp/gateway_test.go

 package corehttp
 
 import (
+	"context"
 	"errors"
 	"io/ioutil"
 	"net/http"
@@ -9,14 +10,15 @@ import (
 	"testing"
 	"time"
 
-	context "context"
 	core "github.com/ipfs/go-ipfs/core"
 	coreunix "github.com/ipfs/go-ipfs/core/coreunix"
+	dag "github.com/ipfs/go-ipfs/merkledag"
 	namesys "github.com/ipfs/go-ipfs/namesys"
 	path "github.com/ipfs/go-ipfs/path"
 	repo "github.com/ipfs/go-ipfs/repo"
 	config "github.com/ipfs/go-ipfs/repo/config"
 	testutil "github.com/ipfs/go-ipfs/thirdparty/testutil"
+
 	id "gx/ipfs/QmcRa2qn6iCmap9bjp8jAwkvYAq13AUfxdY3rrYiaJbLum/go-libp2p/p2p/protocol/identify"
 	ci "gx/ipfs/QmfWDLQjGjVe4fr5CoztYW2DYYjRysMJrFe1RCsXLPTf46/go-libp2p-crypto"
 )
@@ -178,11 +180,13 @@ func TestIPNSHostnameRedirect(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+
 	_, dagn2, err := coreunix.AddWrapped(n, strings.NewReader("_"), "index.html")
 	if err != nil {
 		t.Fatal(err)
 	}
-	dagn1.AddNodeLink("foo", dagn2)
+
+	dagn1.(*dag.ProtoNode).AddNodeLink("foo", dagn2)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -197,7 +201,7 @@ func TestIPNSHostnameRedirect(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	k := dagn1.Key()
+	k := dagn1.Cid()
 	t.Logf("k: %s\n", k)
 	ns["/ipns/example.net"] = path.FromString("/ipfs/" + k.String())
 
@@ -268,8 +272,8 @@ func TestIPNSHostnameBacklinks(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	dagn2.AddNodeLink("bar", dagn3)
-	dagn1.AddNodeLink("foo? #<'", dagn2)
+	dagn2.(*dag.ProtoNode).AddNodeLink("bar", dagn3)
+	dagn1.(*dag.ProtoNode).AddNodeLink("foo? #<'", dagn2)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -287,7 +291,7 @@ func TestIPNSHostnameBacklinks(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	k := dagn1.Key()
+	k := dagn1.Cid()
 	t.Logf("k: %s\n", k)
 	ns["/ipns/example.net"] = path.FromString("/ipfs/" + k.String())
 

core/coreunix/add.go

@@ -104,7 +104,7 @@ type Adder struct {
 	Silent     bool
 	Wrap       bool
 	Chunker    string
-	root       *dag.ProtoNode
+	root       node.Node
 	mr         *mfs.Root
 	unlocker   bs.Unlocker
 	tempRoot   *cid.Cid
@@ -133,7 +133,7 @@ func (adder Adder) add(reader io.Reader) (node.Node, error) {
 	return balanced.BalancedLayout(params.New(chnk))
 }
 
-func (adder *Adder) RootNode() (*dag.ProtoNode, error) {
+func (adder *Adder) RootNode() (node.Node, error) {
 	// for memoizing
 	if adder.root != nil {
 		return adder.root, nil
@@ -151,12 +151,7 @@ func (adder *Adder) RootNode() (*dag.ProtoNode, error) {
 			return nil, err
 		}
 
-		pbnd, ok := nd.(*dag.ProtoNode)
-		if !ok {
-			return nil, dag.ErrNotProtobuf
-		}
-
-		root = pbnd
+		root = nd
 	}
 
 	adder.root = root
@@ -189,7 +184,7 @@ func (adder *Adder) PinRoot() error {
 	return adder.pinning.Flush()
 }
 
-func (adder *Adder) Finalize() (*dag.ProtoNode, error) {
+func (adder *Adder) Finalize() (node.Node, error) {
 	root := adder.mr.GetValue()
 
 	// cant just call adder.RootNode() here as we need the name for printing
@@ -311,7 +306,7 @@ func AddR(n *core.IpfsNode, root string) (key string, err error) {
 // to preserve the filename.
 // Returns the path of the added file ("<dir hash>/filename"), the DAG node of
 // the directory, and and error if any.
-func AddWrapped(n *core.IpfsNode, r io.Reader, filename string) (string, *dag.ProtoNode, error) {
+func AddWrapped(n *core.IpfsNode, r io.Reader, filename string) (string, node.Node, error) {
 	file := files.NewReaderFile(filename, filename, ioutil.NopCloser(r), nil)
 	fileAdder, err := NewAdder(n.Context(), n.Pinning, n.Blockstore, n.DAG)
 	if err != nil {

core/coreunix/cat.go

@@ -4,7 +4,6 @@ import (
 	"context"
 
 	core "github.com/ipfs/go-ipfs/core"
-	dag "github.com/ipfs/go-ipfs/merkledag"
 	path "github.com/ipfs/go-ipfs/path"
 	uio "github.com/ipfs/go-ipfs/unixfs/io"
 )
@@ -15,10 +14,5 @@ func Cat(ctx context.Context, n *core.IpfsNode, pstr string) (*uio.DagReader, er
 		return nil, err
 	}
 
-	dnpb, ok := dagNode.(*dag.ProtoNode)
-	if !ok {
-		return nil, dag.ErrNotProtobuf
-	}
-
-	return uio.NewDagReader(ctx, dnpb, n.DAG)
+	return uio.NewDagReader(ctx, dagNode, n.DAG)
 }

mfs/dir.go

@@ -89,7 +89,7 @@ func (d *Directory) flushCurrentNode() (*dag.ProtoNode, error) {
 	return d.node.Copy(), nil
 }
 
-func (d *Directory) updateChild(name string, nd *dag.ProtoNode) error {
+func (d *Directory) updateChild(name string, nd node.Node) error {
 	err := d.node.RemoveNodeLink(name)
 	if err != nil && err != dag.ErrNotFound {
 		return err
@@ -121,28 +121,40 @@ func (d *Directory) childNode(name string) (FSNode, error) {
 }
 
 // cacheNode caches a node into d.childDirs or d.files and returns the FSNode.
-func (d *Directory) cacheNode(name string, nd *dag.ProtoNode) (FSNode, error) {
-	i, err := ft.FromBytes(nd.Data())
-	if err != nil {
-		return nil, err
-	}
+func (d *Directory) cacheNode(name string, nd node.Node) (FSNode, error) {
+	switch nd := nd.(type) {
+	case *dag.ProtoNode:
+		i, err := ft.FromBytes(nd.Data())
+		if err != nil {
+			return nil, err
+		}
 
-	switch i.GetType() {
-	case ufspb.Data_Directory:
-		ndir := NewDirectory(d.ctx, name, nd, d, d.dserv)
-		d.childDirs[name] = ndir
-		return ndir, nil
-	case ufspb.Data_File, ufspb.Data_Raw, ufspb.Data_Symlink:
+		switch i.GetType() {
+		case ufspb.Data_Directory:
+			ndir := NewDirectory(d.ctx, name, nd, d, d.dserv)
+			d.childDirs[name] = ndir
+			return ndir, nil
+		case ufspb.Data_File, ufspb.Data_Raw, ufspb.Data_Symlink:
+			nfi, err := NewFile(name, nd, d, d.dserv)
+			if err != nil {
+				return nil, err
+			}
+			d.files[name] = nfi
+			return nfi, nil
+		case ufspb.Data_Metadata:
+			return nil, ErrNotYetImplemented
+		default:
+			return nil, ErrInvalidChild
+		}
+	case *dag.RawNode:
 		nfi, err := NewFile(name, nd, d, d.dserv)
 		if err != nil {
 			return nil, err
 		}
 		d.files[name] = nfi
 		return nfi, nil
-	case ufspb.Data_Metadata:
-		return nil, ErrNotYetImplemented
 	default:
-		return nil, ErrInvalidChild
+		return nil, fmt.Errorf("unrecognized node type in cache node")
 	}
 }
 
@@ -162,8 +174,8 @@ func (d *Directory) Uncache(name string) {
 
 // childFromDag searches through this directories dag node for a child link
 // with the given name
-func (d *Directory) childFromDag(name string) (*dag.ProtoNode, error) {
-	pbn, err := d.node.GetLinkedProtoNode(d.ctx, d.dserv, name)
+func (d *Directory) childFromDag(name string) (node.Node, error) {
+	pbn, err := d.node.GetLinkedNode(d.ctx, d.dserv, name)
 	switch err {
 	case nil:
 		return pbn, nil
@@ -249,7 +261,7 @@ func (d *Directory) List() ([]NodeListing, error) {
 			return nil, err
 		}
 
-		child.Hash = nd.Key().B58String()
+		child.Hash = nd.Cid().String()
 
 		out = append(out, child)
 	}
@@ -385,7 +397,7 @@ func (d *Directory) Path() string {
 	return out
 }
 
-func (d *Directory) GetNode() (*dag.ProtoNode, error) {
+func (d *Directory) GetNode() (node.Node, error) {
 	d.lock.Lock()
 	defer d.lock.Unlock()
 

mfs/file.go

@@ -9,6 +9,8 @@ import (
 	dag "github.com/ipfs/go-ipfs/merkledag"
 	ft "github.com/ipfs/go-ipfs/unixfs"
 	mod "github.com/ipfs/go-ipfs/unixfs/mod"
+
+	node "gx/ipfs/QmZx42H5khbVQhV5odp66TApShV4XCujYazcvYduZ4TroB/go-ipld-node"
 )
 
 type File struct {
@@ -19,12 +21,12 @@ type File struct {
 	desclock sync.RWMutex
 
 	dserv  dag.DAGService
-	node   *dag.ProtoNode
+	node   node.Node
 	nodelk sync.Mutex
 }
 
 // NewFile returns a NewFile object with the given parameters
-func NewFile(name string, node *dag.ProtoNode, parent childCloser, dserv dag.DAGService) (*File, error) {
+func NewFile(name string, node node.Node, parent childCloser, dserv dag.DAGService) (*File, error) {
 	return &File{
 		dserv:  dserv,
 		parent: parent,
@@ -44,18 +46,23 @@ func (fi *File) Open(flags int, sync bool) (FileDescriptor, error) {
 	node := fi.node
 	fi.nodelk.Unlock()
 
-	fsn, err := ft.FSNodeFromBytes(node.Data())
-	if err != nil {
-		return nil, err
-	}
-
-	switch fsn.Type {
-	default:
-		return nil, fmt.Errorf("unsupported fsnode type for 'file'")
-	case ft.TSymlink:
-		return nil, fmt.Errorf("symlinks not yet supported")
-	case ft.TFile, ft.TRaw:
-		// OK case
+	switch node := node.(type) {
+	case *dag.ProtoNode:
+		fsn, err := ft.FSNodeFromBytes(node.Data())
+		if err != nil {
+			return nil, err
+		}
+
+		switch fsn.Type {
+		default:
+			return nil, fmt.Errorf("unsupported fsnode type for 'file'")
+		case ft.TSymlink:
+			return nil, fmt.Errorf("symlinks not yet supported")
+		case ft.TFile, ft.TRaw:
+			// OK case
+		}
+	case *dag.RawNode:
+		// Ok as well.
 	}
 
 	switch flags {
@@ -85,16 +92,22 @@ func (fi *File) Open(flags int, sync bool) (FileDescriptor, error) {
 func (fi *File) Size() (int64, error) {
 	fi.nodelk.Lock()
 	defer fi.nodelk.Unlock()
-	pbd, err := ft.FromBytes(fi.node.Data())
-	if err != nil {
-		return 0, err
+	switch nd := fi.node.(type) {
+	case *dag.ProtoNode:
+		pbd, err := ft.FromBytes(nd.Data())
+		if err != nil {
+			return 0, err
+		}
+		return int64(pbd.GetFilesize()), nil
+	case *dag.RawNode:
+		return int64(len(nd.RawData())), nil
+	default:
+		return 0, fmt.Errorf("unrecognized node type in mfs/file.Size()")
 	}
-
-	return int64(pbd.GetFilesize()), nil
 }
 
 // GetNode returns the dag node associated with this file
-func (fi *File) GetNode() (*dag.ProtoNode, error) {
+func (fi *File) GetNode() (node.Node, error) {
 	fi.nodelk.Lock()
 	defer fi.nodelk.Unlock()
 	return fi.node, nil

mfs/mfs_test.go

@@ -794,7 +794,12 @@ func TestFlushing(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	fsnode, err := ft.FSNodeFromBytes(rnd.Data())
+	pbrnd, ok := rnd.(*dag.ProtoNode)
+	if !ok {
+		t.Fatal(dag.ErrNotProtobuf)
+	}
+
+	fsnode, err := ft.FSNodeFromBytes(pbrnd.Data())
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -803,10 +808,10 @@ func TestFlushing(t *testing.T) {
 		t.Fatal("root wasnt a directory")
 	}
 
-	rnk := rnd.Key()
+	rnk := rnd.Cid()
 	exp := "QmWMVyhTuyxUrXX3ynz171jq76yY3PktfY9Bxiph7b9ikr"
-	if rnk.B58String() != exp {
-		t.Fatalf("dag looks wrong, expected %s, but got %s", exp, rnk.B58String())
+	if rnk.String() != exp {
+		t.Fatalf("dag looks wrong, expected %s, but got %s", exp, rnk.String())
 	}
 }
 

mfs/system.go

@@ -10,6 +10,7 @@
 package mfs
 
 import (
+	"context"
 	"errors"
 	"sync"
 	"time"
@@ -17,9 +18,9 @@ import (
 	dag "github.com/ipfs/go-ipfs/merkledag"
 	ft "github.com/ipfs/go-ipfs/unixfs"
 
-	context "context"
 	logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log"
 	cid "gx/ipfs/QmXUuRadqDq5BuFWzVU6VuKaSjTcNm1gNCtLvvP1TJCW4z/go-cid"
+	node "gx/ipfs/QmZx42H5khbVQhV5odp66TApShV4XCujYazcvYduZ4TroB/go-ipld-node"
 )
 
 var ErrNotExist = errors.New("no such rootfs")
@@ -41,7 +42,7 @@ const (
 
 // FSNode represents any node (directory, root, or file) in the mfs filesystem
 type FSNode interface {
-	GetNode() (*dag.ProtoNode, error)
+	GetNode() (node.Node, error)
 	Flush() error
 	Type() NodeType
 }

test/sharness/t0040-add-and-cat.sh

@@ -126,6 +126,21 @@ test_add_cat_5MB() {
     '
 }
 
+test_add_cat_raw() {
+	test_expect_success "add a small file with raw-leaves" '
+		echo "foobar" > afile &&
+		HASH=$(ipfs add -q --raw-leaves afile)
+	'
+
+	test_expect_success "cat that small file" '
+		ipfs cat $HASH > afile_out
+	'
+
+	test_expect_success "make sure it looks good" '
+		test_cmp afile afile_out
+	'
+}
+
 test_add_cat_expensive() {
     test_expect_success EXPENSIVE "generate 100MB file using go-random" '
     	random 104857600 42 >mountdir/bigfile
@@ -392,17 +407,21 @@ test_add_named_pipe " Post http://$API_ADDR/api/v0/add?encoding=json&progress=tr
 
 test_add_pwd_is_symlink
 
+test_add_cat_raw
+
 test_kill_ipfs_daemon
 
 # should work offline
 
 test_add_cat_file
 
+test_add_cat_raw
+
 test_expect_success "ipfs add --only-hash succeeds" '
     echo "unknown content for only-hash" | ipfs add --only-hash -q > oh_hash
 '
 
-#TODO: this doesn't work when online hence separated out from test_add_cat_file
+#TODO: this doesnt work when online hence separated out from test_add_cat_file
 test_expect_success "ipfs cat file fails" '
     test_must_fail ipfs cat $(cat oh_hash)
 '