Merge pull request #3325 from ipfs/feat/cbor-ipld

Implement cbor ipld nodes and a first pass at the 'dag' command

core/commands/block.go

@@ -2,7 +2,6 @@ package commands
 
 import (
 	"bytes"
-	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -168,10 +167,6 @@ func getBlockForKey(req cmds.Request, skey string) (blocks.Block, error) {
 		return nil, err
 	}
 
-	if !u.IsValidHash(skey) {
-		return nil, errors.New("Not a valid hash")
-	}
-
 	c, err := cid.Decode(skey)
 	if err != nil {
 		return nil, err

core/commands/dag/dag.go

+package dagcmd
+
+import (
+	"fmt"
+	"io"
+	"strings"
+
+	cmds "github.com/ipfs/go-ipfs/commands"
+
+	ipldcbor "gx/ipfs/QmRcAVqrbY5wryx7hfNLtiUZbCcstzaJL7YJFBboitcqWF/go-ipld-cbor"
+	node "gx/ipfs/QmU7bFWQ793qmvNy7outdCaMfSDNk8uqhx4VNrxYj5fj5g/go-ipld-node"
+	cid "gx/ipfs/QmXfiyr2RWEXpVDdaYnD2HNiBk6UBddsvEP4RPfXb6nGqY/go-cid"
+)
+
+var DagCmd = &cmds.Command{
+	Helptext: cmds.HelpText{
+		Tagline: "Interact with ipld dag objects.",
+		ShortDescription: `
+'ipfs dag' is used for creating and manipulating dag objects.
+
+This subcommand is currently an experimental feature, but it is intended
+to deprecate and replace the existing 'ipfs object' command moving forward.
+		`,
+	},
+	Subcommands: map[string]*cmds.Command{
+		"put": DagPutCmd,
+		"get": DagGetCmd,
+	},
+}
+
+type OutputObject struct {
+	Cid *cid.Cid
+}
+
+var DagPutCmd = &cmds.Command{
+	Helptext: cmds.HelpText{
+		Tagline: "Add a dag node to ipfs.",
+		ShortDescription: `
+'ipfs dag put' accepts input from a file or stdin and parses it
+into an object of the specified format.
+`,
+	},
+	Arguments: []cmds.Argument{
+		cmds.FileArg("object data", true, false, "The object to put").EnableStdin(),
+	},
+	Options: []cmds.Option{
+		cmds.StringOption("format", "f", "Format that the object will be added as.").Default("cbor"),
+		cmds.StringOption("input-enc", "Format that the input object will be.").Default("json"),
+	},
+	Run: func(req cmds.Request, res cmds.Response) {
+		n, err := req.InvocContext().GetNode()
+		if err != nil {
+			res.SetError(err, cmds.ErrNormal)
+			return
+		}
+
+		fi, err := req.Files().NextFile()
+		if err != nil {
+			res.SetError(err, cmds.ErrNormal)
+			return
+		}
+
+		ienc, _, _ := req.Option("input-enc").String()
+		format, _, _ := req.Option("format").String()
+
+		switch ienc {
+		case "json":
+			nd, err := convertJsonToType(fi, format)
+			if err != nil {
+				res.SetError(err, cmds.ErrNormal)
+				return
+			}
+
+			c, err := n.DAG.Add(nd)
+			if err != nil {
+				res.SetError(err, cmds.ErrNormal)
+				return
+			}
+
+			res.SetOutput(&OutputObject{Cid: c})
+			return
+		default:
+			res.SetError(fmt.Errorf("unrecognized input encoding: %s", ienc), cmds.ErrNormal)
+			return
+		}
+	},
+	Type: OutputObject{},
+	Marshalers: cmds.MarshalerMap{
+		cmds.Text: func(res cmds.Response) (io.Reader, error) {
+			oobj, ok := res.Output().(*OutputObject)
+			if !ok {
+				return nil, fmt.Errorf("expected a different object in marshaler")
+			}
+
+			return strings.NewReader(oobj.Cid.String()), nil
+		},
+	},
+}
+
+var DagGetCmd = &cmds.Command{
+	Helptext: cmds.HelpText{
+		Tagline: "Get a dag node from ipfs.",
+		ShortDescription: `
+'ipfs dag get' fetches a dag node from ipfs and prints it out in the specifed format.
+`,
+	},
+	Arguments: []cmds.Argument{
+		cmds.StringArg("cid", true, false, "The cid of the object to get").EnableStdin(),
+	},
+	Run: func(req cmds.Request, res cmds.Response) {
+		n, err := req.InvocContext().GetNode()
+		if err != nil {
+			res.SetError(err, cmds.ErrNormal)
+			return
+		}
+
+		c, err := cid.Decode(req.Arguments()[0])
+		if err != nil {
+			res.SetError(err, cmds.ErrNormal)
+			return
+		}
+
+		obj, err := n.DAG.Get(req.Context(), c)
+		if err != nil {
+			res.SetError(err, cmds.ErrNormal)
+			return
+		}
+
+		res.SetOutput(obj)
+	},
+}
+
+func convertJsonToType(r io.Reader, format string) (node.Node, error) {
+	switch format {
+	case "cbor", "dag-cbor":
+		return ipldcbor.FromJson(r)
+	case "dag-pb", "protobuf":
+		return nil, fmt.Errorf("protobuf handling in 'dag' command not yet implemented")
+	default:
+		return nil, fmt.Errorf("unknown target format: %s", format)
+	}
+}

core/commands/root.go

@@ -5,6 +5,7 @@ import (
 	"strings"
 
 	cmds "github.com/ipfs/go-ipfs/commands"
+	dag "github.com/ipfs/go-ipfs/core/commands/dag"
 	files "github.com/ipfs/go-ipfs/core/commands/files"
 	ocmd "github.com/ipfs/go-ipfs/core/commands/object"
 	unixfs "github.com/ipfs/go-ipfs/core/commands/unixfs"
@@ -86,6 +87,7 @@ var rootSubcommands = map[string]*cmds.Command{
 	"cat":       CatCmd,
 	"commands":  CommandsDaemonCmd,
 	"config":    ConfigCmd,
+	"dag":       dag.DagCmd,
 	"dht":       DhtCmd,
 	"diag":      DiagCmd,
 	"dns":       DNSCmd,

merkledag/merkledag.go

@@ -11,6 +11,7 @@ import (
 	bserv "github.com/ipfs/go-ipfs/blockservice"
 	offline "github.com/ipfs/go-ipfs/exchange/offline"
 
+	ipldcbor "gx/ipfs/QmRcAVqrbY5wryx7hfNLtiUZbCcstzaJL7YJFBboitcqWF/go-ipld-cbor"
 	logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log"
 	node "gx/ipfs/QmU7bFWQ793qmvNy7outdCaMfSDNk8uqhx4VNrxYj5fj5g/go-ipld-node"
 	cid "gx/ipfs/QmXfiyr2RWEXpVDdaYnD2HNiBk6UBddsvEP4RPfXb6nGqY/go-cid"
@@ -105,6 +106,8 @@ func decodeBlock(b blocks.Block) (node.Node, error) {
 		return decnd, nil
 	case cid.Raw:
 		return NewRawNode(b.RawData()), nil
+	case cid.CBOR:
+		return ipldcbor.Decode(b.RawData())
 	default:
 		return nil, fmt.Errorf("unrecognized object type: %s", c.Type())
 	}

package.json

@@ -277,9 +277,10 @@
       "version": "0.3.2"
     },
     {
-      "hash": "QmQKEgGgYCDyk8VNY6A65FpuE4YwbspvjXHco1rdb75PVc",
-      "name": "go-libp2p-routing",
-      "version": "2.2.2"
+      "author": "whyrusleeping",
+      "hash": "QmRcAVqrbY5wryx7hfNLtiUZbCcstzaJL7YJFBboitcqWF",
+      "name": "go-ipld-cbor",
+      "version": "0.3.0"
     }
   ],
   "gxVersion": "0.4.0",

test/sharness/t0053-dag.sh

+#!/bin/sh
+#
+# Copyright (c) 2016 Jeromy Johnson
+# MIT Licensed; see the LICENSE file in this repository.
+#
+
+test_description="Test dag command"
+
+. lib/test-lib.sh
+
+test_init_ipfs
+
+test_expect_success "make a few test files" '
+	echo "foo" > file1 &&
+	echo "bar" > file2 &&
+	echo "baz" > file3 &&
+	echo "qux" > file4 &&
+	HASH1=$(ipfs add -q file1) &&
+	HASH2=$(ipfs add -q file2) &&
+	HASH3=$(ipfs add -q file3) &&
+	HASH4=$(ipfs add -q file4)
+'
+
+test_expect_success "make an ipld object in json" '
+	printf "{\"hello\":\"world\",\"cats\":[{\"/\":\"%s\"},{\"water\":{\"/\":\"%s\"}}],\"magic\":{\"/\":\"%s\"}}" $HASH1 $HASH2 $HASH3 > ipld_object
+'
+
+test_dag_cmd() {
+	test_expect_success "can add an ipld object" '
+		IPLDHASH=$(cat ipld_object | ipfs dag put)
+	'
+
+	test_expect_success "output looks correct" '
+		EXPHASH="zdpuApvChR5xM7ttbQmpmtna7wcShHi4gPyxUcWbB7nh8K7cN"
+		test $EXPHASH = $IPLDHASH
+	'
+
+	test_expect_success "various path traversals work" '
+		ipfs cat $IPLDHASH/cats/0 > out1 &&
+		ipfs cat $IPLDHASH/cats/1/water > out2 &&
+		ipfs cat $IPLDHASH/magic > out3
+	'
+
+	test_expect_success "outputs look correct" '
+		test_cmp file1 out1 &&
+		test_cmp file2 out2 &&
+		test_cmp file3 out3
+	'
+}
+
+# should work offline
+test_dag_cmd
+
+# should work online
+test_launch_ipfs_daemon
+test_dag_cmd
+test_kill_ipfs_daemon
+
+test_done