Merge pull request #3557 from ipfs/kevina/raw-leaves-ls-fix

Fix "ipfs ls" so it works correctly with raw leaves.

core/commands/ls.go

@@ -6,8 +6,10 @@ import (
 	"io"
 	"text/tabwriter"
 
+	blockservice "github.com/ipfs/go-ipfs/blockservice"
 	cmds "github.com/ipfs/go-ipfs/commands"
 	core "github.com/ipfs/go-ipfs/core"
+	offline "github.com/ipfs/go-ipfs/exchange/offline"
 	merkledag "github.com/ipfs/go-ipfs/merkledag"
 	path "github.com/ipfs/go-ipfs/path"
 	unixfs "github.com/ipfs/go-ipfs/unixfs"
@@ -71,6 +73,13 @@ The JSON output contains type information.
 			return
 		}
 
+		dserv := nd.DAG
+		if !resolve {
+			offlineexch := offline.Exchange(nd.Blockstore)
+			bserv := blockservice.New(nd.Blockstore, offlineexch)
+			dserv = merkledag.NewDAGService(bserv)
+		}
+
 		paths := req.Arguments()
 
 		var dagnodes []node.Node
@@ -101,39 +110,19 @@ The JSON output contains type information.
 				Links: make([]LsLink, len(dagnode.Links())),
 			}
 			for j, link := range dagnode.Links() {
-				var linkNode *merkledag.ProtoNode
 				t := unixfspb.Data_DataType(-1)
-				linkKey := link.Cid
-				if ok, err := nd.Blockstore.Has(linkKey); ok && err == nil {
-					b, err := nd.Blockstore.Get(linkKey)
-					if err != nil {
-						res.SetError(err, cmds.ErrNormal)
-						return
-					}
-					linkNode, err = merkledag.DecodeProtobuf(b.RawData())
-					if err != nil {
-						res.SetError(err, cmds.ErrNormal)
-						return
-					}
-				}
-
-				if linkNode == nil && resolve {
-					nd, err := link.GetNode(req.Context(), nd.DAG)
-					if err != nil {
-						res.SetError(err, cmds.ErrNormal)
-						return
-					}
 
-					pbnd, ok := nd.(*merkledag.ProtoNode)
-					if !ok {
-						res.SetError(merkledag.ErrNotProtobuf, cmds.ErrNormal)
-						return
-					}
-
-					linkNode = pbnd
+				linkNode, err := link.GetNode(req.Context(), dserv)
+				if err == merkledag.ErrNotFound && !resolve {
+					// not an error
+					linkNode = nil
+				} else if err != nil {
+					res.SetError(err, cmds.ErrNormal)
+					return
 				}
-				if linkNode != nil {
-					d, err := unixfs.FromBytes(linkNode.Data())
+
+				if pn, ok := linkNode.(*merkledag.ProtoNode); ok {
+					d, err := unixfs.FromBytes(pn.Data())
 					if err != nil {
 						res.SetError(err, cmds.ErrNormal)
 						return

test/sharness/t0040-add-and-cat.sh

@@ -315,22 +315,73 @@ test_expect_success "ipfs cat with built input output looks good" '
 	test_cmp expected actual
 '
 
-test_expect_success "'ipfs add -r' succeeds" '
-	mkdir mountdir/planets &&
-	echo "Hello Mars!" >mountdir/planets/mars.txt &&
-	echo "Hello Venus!" >mountdir/planets/venus.txt &&
-	ipfs add -r mountdir/planets >actual
-'
+add_directory() {
+    EXTRA_ARGS=$1
+
+    test_expect_success "'ipfs add -r $EXTRA_ARGS' succeeds" '
+            mkdir mountdir/planets &&
+            echo "Hello Mars!" >mountdir/planets/mars.txt &&
+            echo "Hello Venus!" >mountdir/planets/venus.txt &&
+            ipfs add -r $EXTRA_ARGS mountdir/planets >actual
+    '
+
+    test_expect_success "'ipfs add -r $EXTRA_ARGS' output looks good" '
+            echo "added $MARS planets/mars.txt" >expected &&
+            echo "added $VENUS planets/venus.txt" >>expected &&
+            echo "added $PLANETS planets" >>expected &&
+            test_cmp expected actual
+    '
+
+    test_expect_success "ipfs cat accept many hashes from built input" '
+            { echo "$MARS"; echo "$VENUS"; } | ipfs cat >actual
+    '
+
+    test_expect_success "ipfs cat output looks good" '
+            cat mountdir/planets/mars.txt mountdir/planets/venus.txt >expected &&
+            test_cmp expected actual
+    '
+
+    test_expect_success "ipfs cat accept many hashes as args" '
+            ipfs cat "$MARS" "$VENUS" >actual
+    '
+
+    test_expect_success "ipfs cat output looks good" '
+            test_cmp expected actual
+    '
+
+    test_expect_success "ipfs cat with both arg and stdin" '
+            echo "$MARS" | ipfs cat "$VENUS" >actual
+    '
+
+    test_expect_success "ipfs cat output looks good" '
+            cat mountdir/planets/venus.txt >expected &&
+            test_cmp expected actual
+    '
+
+    test_expect_success "ipfs cat with two args and stdin" '
+            echo "$MARS" | ipfs cat "$VENUS" "$VENUS" >actual
+    '
+
+    test_expect_success "ipfs cat output looks good" '
+            cat mountdir/planets/venus.txt mountdir/planets/venus.txt >expected &&
+            test_cmp expected actual
+    '
+
+    test_expect_success "cleanup" '
+            rm -r mountdir/planets
+    '
+}
+
+PLANETS="QmWSgS32xQEcXMeqd3YPJLrNBLSdsfYCep2U7CFkyrjXwY"
+MARS="QmPrrHqJzto9m7SyiRzarwkqPcCSsKR2EB1AyqJfe8L8tN"
+VENUS="QmU5kp3BH3B8tnWUU2Pikdb2maksBNkb92FHRr56hyghh4"
+add_directory
+
+PLANETS="QmfWfQfKCY5Ukv9peBbxM5vqWM9BzmqUSXvdCgjT2wsiBT"
+MARS="zb2rhZdTkQNawVajsTNiYc9cTPHqgLdJVvBRkZok9RjkgQYRU"
+VENUS="zb2rhn6TGvnUaMAg4VV4y9HVx5W42HihcH4jsyrDv8mkepFqq"
+add_directory '--raw-leaves'
 
-test_expect_success "'ipfs add -r' output looks good" '
-	PLANETS="QmWSgS32xQEcXMeqd3YPJLrNBLSdsfYCep2U7CFkyrjXwY" &&
-	MARS="QmPrrHqJzto9m7SyiRzarwkqPcCSsKR2EB1AyqJfe8L8tN" &&
-	VENUS="QmU5kp3BH3B8tnWUU2Pikdb2maksBNkb92FHRr56hyghh4" &&
-	echo "added $MARS planets/mars.txt" >expected &&
-	echo "added $VENUS planets/venus.txt" >>expected &&
-	echo "added $PLANETS planets" >>expected &&
-	test_cmp expected actual
-'
 
 test_expect_success "'ipfs add -rn' succeeds" '
 	mkdir -p mountdir/moons/jupiter &&
@@ -357,42 +408,6 @@ test_expect_success "'ipfs add -rn' output looks good" '
 	test_cmp expected actual
 '
 
-test_expect_success "ipfs cat accept many hashes from built input" '
-	{ echo "$MARS"; echo "$VENUS"; } | ipfs cat >actual
-'
-
-test_expect_success "ipfs cat output looks good" '
-	cat mountdir/planets/mars.txt mountdir/planets/venus.txt >expected &&
-	test_cmp expected actual
-'
-
-test_expect_success "ipfs cat accept many hashes as args" '
-	ipfs cat "$MARS" "$VENUS" >actual
-'
-
-test_expect_success "ipfs cat output looks good" '
-	test_cmp expected actual
-'
-
-test_expect_success "ipfs cat with both arg and stdin" '
-	echo "$MARS" | ipfs cat "$VENUS" >actual
-'
-
-test_expect_success "ipfs cat output looks good" '
-	cat mountdir/planets/venus.txt >expected &&
-	test_cmp expected actual
-'
-
-test_expect_success "ipfs cat with two args and stdin" '
-	echo "$MARS" | ipfs cat "$VENUS" "$VENUS" >actual
-'
-
-test_expect_success "ipfs cat output looks good" '
-	cat mountdir/planets/venus.txt mountdir/planets/venus.txt >expected &&
-	test_cmp expected actual
-'
-
-
 test_expect_success "go-random is installed" '
     type random
 '

test/sharness/t0045-ls.sh

@@ -90,12 +90,80 @@ test_ls_cmd() {
 	'
 }
 
+test_ls_cmd_raw_leaves() {
+	test_expect_success "'ipfs add -r --raw-leaves' then 'ipfs ls' works as expected" '
+		mkdir -p somedir &&
+		echo bar > somedir/foo &&
+		ipfs add --raw-leaves -r somedir/ > /dev/null &&
+		ipfs ls QmThNTdtKaVoCVrYmM5EBS6U3S5vfKFue2TxbxxAxRcKKE > ls-actual
+		echo "zb2rhf6GzX4ckKZtjy8yy8iyq1KttCrRyqDedD6xubhY3sw2F 4 foo" > ls-expect
+		test_cmp ls-actual ls-expect
+	'
+}
+
 # should work offline
 test_ls_cmd
+test_ls_cmd_raw_leaves
 
 # should work online
 test_launch_ipfs_daemon
 test_ls_cmd
+test_ls_cmd_raw_leaves
+test_kill_ipfs_daemon
+
+#
+# test for ls --resolve-type=false
+#
+
+test_expect_success "'ipfs add -r' succeeds" '
+	mkdir adir &&
+	# note: not using a seed as the files need to have truly random content
+	random 1000 > adir/file1 &&
+	random 1000 > adir/file2 &&
+	ipfs add --pin=false -q -r adir > adir-hashes
+'
+
+test_expect_success "get hashes from add output" '
+	FILE=`head -1 adir-hashes` &&
+	DIR=`tail -1 adir-hashes` &&
+	test "$FILE" -a "$DIR"
+'
+
+test_expect_success "remove a file in dir" '
+	ipfs block rm $FILE
+'
+
+test_expect_success "'ipfs ls --resolve-type=false ' ok" '
+	ipfs ls --resolve-type=false $DIR > /dev/null
+'
+
+test_expect_success "'ipfs ls' fails" '
+	test_must_fail ipfs ls $DIR
+'
+
+test_launch_ipfs_daemon --offline
+
+test_expect_success "'ipfs ls --resolve-type=false' ok" '
+	ipfs ls --resolve-type=false $DIR > /dev/null
+'
+
+test_expect_success "'ipfs ls' fails" '
+	test_must_fail ipfs ls $DIR
+'
+
+test_kill_ipfs_daemon
+
+test_launch_ipfs_daemon
+
+# now we try `ipfs ls --resolve-type=false` with the daemon online It
+# should not even attempt to retrieve the file from the network.  If
+# it does it should eventually fail as the content is random and
+# should not exist on the network, but we don't want to wait for a
+# timeout so we will kill the request after a few seconds
+test_expect_success "'ipfs ls --resolve-type=false' ok and does not hang" '
+	go-timeout 2 ipfs ls --resolve-type=false $DIR
+'
+
 test_kill_ipfs_daemon
 
 test_done