Merge pull request #4133 from ipfs/feat/improve-dag-api

Improve dag API

core/commands/dag/dag.go

 package dagcmd
 
 import (
+	"bytes"
 	"fmt"
 	"io"
+	"math"
 	"strings"
 
 	cmds "github.com/ipfs/go-ipfs/commands"
@@ -11,6 +13,7 @@ import (
 	pin "github.com/ipfs/go-ipfs/pin"
 
 	cid "gx/ipfs/QmTprEaAA2A9bst5XH7exuyi5KzNMK3SEDNN8rBDnKWcUS/go-cid"
+	mh "gx/ipfs/QmU9a9NV9RdPNwZQDYd5uKsm6N6LJLSvLbywDDYFbaaC6P/go-multihash"
 )
 
 var DagCmd = &cmds.Command{
@@ -24,15 +27,23 @@ to deprecate and replace the existing 'ipfs object' command moving forward.
 		`,
 	},
 	Subcommands: map[string]*cmds.Command{
-		"put": DagPutCmd,
-		"get": DagGetCmd,
+		"put":     DagPutCmd,
+		"get":     DagGetCmd,
+		"resolve": DagResolveCmd,
 	},
 }
 
+// OutputObject is the output type of 'dag put' command
 type OutputObject struct {
 	Cid *cid.Cid
 }
 
+// ResolveOutput is the output type of 'dag resolve' command
+type ResolveOutput struct {
+	Cid     *cid.Cid
+	RemPath string
+}
+
 var DagPutCmd = &cmds.Command{
 	Helptext: cmds.HelpText{
 		Tagline: "Add a dag node to ipfs.",
@@ -48,6 +59,7 @@ into an object of the specified format.
 		cmds.StringOption("format", "f", "Format that the object will be added as.").Default("cbor"),
 		cmds.StringOption("input-enc", "Format that the input object will be.").Default("json"),
 		cmds.BoolOption("pin", "Pin this object when adding.").Default(false),
+		cmds.StringOption("hash", "Hash function to use").Default(""),
 	},
 	Run: func(req cmds.Request, res cmds.Response) {
 		n, err := req.InvocContext().GetNode()
@@ -64,17 +76,31 @@ into an object of the specified format.
 
 		ienc, _, _ := req.Option("input-enc").String()
 		format, _, _ := req.Option("format").String()
+		hash, _, err := req.Option("hash").String()
 		dopin, _, err := req.Option("pin").Bool()
 		if err != nil {
 			res.SetError(err, cmds.ErrNormal)
 			return
 		}
 
+		// mhType tells inputParser which hash should be used. MaxUint64 means 'use
+		// default hash' (sha256 for cbor, sha1 for git..)
+		mhType := uint64(math.MaxUint64)
+
+		if hash != "" {
+			var ok bool
+			mhType, ok = mh.Names[hash]
+			if !ok {
+				res.SetError(fmt.Errorf("%s in not a valid multihash name", hash), cmds.ErrNormal)
+				return
+			}
+		}
+
 		if dopin {
 			defer n.Blockstore.PinLock().Unlock()
 		}
 
-		nds, err := coredag.ParseInputs(ienc, format, fi)
+		nds, err := coredag.ParseInputs(ienc, format, fi, mhType, -1)
 		if err != nil {
 			res.SetError(err, cmds.ErrNormal)
 			return
@@ -166,3 +192,55 @@ var DagGetCmd = &cmds.Command{
 		res.SetOutput(out)
 	},
 }
+
+// DagResolveCmd returns address of highest block within a path and a path remainder
+var DagResolveCmd = &cmds.Command{
+	Helptext: cmds.HelpText{
+		Tagline: "Resolve ipld block",
+		ShortDescription: `
+'ipfs dag resolve' fetches a dag node from ipfs, prints it's address and remaining path.
+`,
+	},
+	Arguments: []cmds.Argument{
+		cmds.StringArg("ref", true, false, "The path to resolve").EnableStdin(),
+	},
+	Run: func(req cmds.Request, res cmds.Response) {
+		n, err := req.InvocContext().GetNode()
+		if err != nil {
+			res.SetError(err, cmds.ErrNormal)
+			return
+		}
+
+		p, err := path.ParsePath(req.Arguments()[0])
+		if err != nil {
+			res.SetError(err, cmds.ErrNormal)
+			return
+		}
+
+		obj, rem, err := n.Resolver.ResolveToLastNode(req.Context(), p)
+		if err != nil {
+			res.SetError(err, cmds.ErrNormal)
+			return
+		}
+
+		res.SetOutput(&ResolveOutput{
+			Cid:     obj.Cid(),
+			RemPath: path.Join(rem),
+		})
+	},
+	Marshalers: cmds.MarshalerMap{
+		cmds.Text: func(res cmds.Response) (io.Reader, error) {
+			output := res.Output().(*ResolveOutput)
+			buf := new(bytes.Buffer)
+			p := output.Cid.String()
+			if output.RemPath != "" {
+				p = path.Join([]string{p, output.RemPath})
+			}
+
+			buf.WriteString(p)
+
+			return buf, nil
+		},
+	},
+	Type: ResolveOutput{},
+}

core/commands/root.go

@@ -165,7 +165,8 @@ var rootROSubcommands = map[string]*cmds.Command{
 	},
 	"dag": &cmds.Command{
 		Subcommands: map[string]*cmds.Command{
-			"get": dag.DagGetCmd,
+			"get":     dag.DagGetCmd,
+			"resolve": dag.DagResolveCmd,
 		},
 	},
 	"refs":    RefsROCmd,

core/coreapi/unixfs_test.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"io"
+	"math"
 	"strings"
 	"testing"
 
@@ -16,7 +17,7 @@ import (
 	config "github.com/ipfs/go-ipfs/repo/config"
 	testutil "github.com/ipfs/go-ipfs/thirdparty/testutil"
 	unixfs "github.com/ipfs/go-ipfs/unixfs"
-	cbor "gx/ipfs/QmXgUVPAxjMLZSyxx818YstJJAoRg3nyPWENmBLVzLtoax/go-ipld-cbor"
+	cbor "gx/ipfs/QmeebqVZeEXBqJ2B4urQWfdhwRRPm84ajnCo8x8pfwbsPM/go-ipld-cbor"
 )
 
 // `echo -n 'hello, world!' | ipfs add`
@@ -277,7 +278,7 @@ func TestLsNonUnixfs(t *testing.T) {
 		t.Error(err)
 	}
 
-	nd, err := cbor.WrapObject(map[string]interface{}{"foo": "bar"})
+	nd, err := cbor.WrapObject(map[string]interface{}{"foo": "bar"}, math.MaxUint64, -1)
 	if err != nil {
 		t.Fatal(err)
 	}

core/coredag/cbor.go

+package coredag
+
+import (
+	"io"
+	"io/ioutil"
+
+	node "gx/ipfs/QmYNyRZJBUYPNrLszFmrBrPJbsBh2vMsefz5gnDpB5M1P6/go-ipld-format"
+	ipldcbor "gx/ipfs/QmeebqVZeEXBqJ2B4urQWfdhwRRPm84ajnCo8x8pfwbsPM/go-ipld-cbor"
+)
+
+func cborJSONParser(r io.Reader, mhType uint64, mhLen int) ([]node.Node, error) {
+	nd, err := ipldcbor.FromJson(r, mhType, mhLen)
+	if err != nil {
+		return nil, err
+	}
+
+	return []node.Node{nd}, nil
+}
+
+func cborRawParser(r io.Reader, mhType uint64, mhLen int) ([]node.Node, error) {
+	data, err := ioutil.ReadAll(r)
+	if err != nil {
+		return nil, err
+	}
+
+	nd, err := ipldcbor.Decode(data, mhType, mhLen)
+	if err != nil {
+		return nil, err
+	}
+
+	return []node.Node{nd}, nil
+}

core/coredag/dagpb.go

+package coredag
+
+import (
+	"io"
+	"io/ioutil"
+	"math"
+
+	"github.com/ipfs/go-ipfs/merkledag"
+
+	cid "gx/ipfs/QmTprEaAA2A9bst5XH7exuyi5KzNMK3SEDNN8rBDnKWcUS/go-cid"
+	mh "gx/ipfs/QmU9a9NV9RdPNwZQDYd5uKsm6N6LJLSvLbywDDYFbaaC6P/go-multihash"
+	node "gx/ipfs/QmYNyRZJBUYPNrLszFmrBrPJbsBh2vMsefz5gnDpB5M1P6/go-ipld-format"
+)
+
+func dagpbJSONParser(r io.Reader, mhType uint64, mhLen int) ([]node.Node, error) {
+	data, err := ioutil.ReadAll(r)
+	if err != nil {
+		return nil, err
+	}
+
+	nd := &merkledag.ProtoNode{}
+
+	err = nd.UnmarshalJSON(data)
+	if err != nil {
+		return nil, err
+	}
+
+	nd.SetPrefix(cidPrefix(mhType, mhLen))
+
+	return []node.Node{nd}, nil
+}
+
+func dagpbRawParser(r io.Reader, mhType uint64, mhLen int) ([]node.Node, error) {
+	data, err := ioutil.ReadAll(r)
+	if err != nil {
+		return nil, err
+	}
+
+	nd, err := merkledag.DecodeProtobuf(data)
+	if err != nil {
+		return nil, err
+	}
+
+	nd.SetPrefix(cidPrefix(mhType, mhLen))
+
+	return []node.Node{nd}, nil
+}
+
+func cidPrefix(mhType uint64, mhLen int) *cid.Prefix {
+	if mhType == math.MaxUint64 {
+		mhType = mh.SHA2_256
+	}
+
+	prefix := &cid.Prefix{
+		MhType:   mhType,
+		MhLength: mhLen,
+		Version:  1,
+		Codec:    cid.DagProtobuf,
+	}
+
+	if mhType == mh.SHA2_256 {
+		prefix.Version = 0
+	}
+
+	return prefix
+}

core/coredag/dagtransl.go

@@ -3,14 +3,12 @@ package coredag
 import (
 	"fmt"
 	"io"
-	"io/ioutil"
 
-	ipldcbor "gx/ipfs/QmXgUVPAxjMLZSyxx818YstJJAoRg3nyPWENmBLVzLtoax/go-ipld-cbor"
 	node "gx/ipfs/QmYNyRZJBUYPNrLszFmrBrPJbsBh2vMsefz5gnDpB5M1P6/go-ipld-format"
 )
 
 // DagParser is function used for parsing stream into Node
-type DagParser func(r io.Reader) ([]node.Node, error)
+type DagParser func(r io.Reader, mhType uint64, mhLen int) ([]node.Node, error)
 
 // FormatParsers is used for mapping format descriptors to DagParsers
 type FormatParsers map[string]DagParser
@@ -22,22 +20,34 @@ type InputEncParsers map[string]FormatParsers
 var DefaultInputEncParsers = InputEncParsers{
 	"json": defaultJSONParsers,
 	"raw":  defaultRawParsers,
+	"cbor": defaultCborParsers,
 }
 
 var defaultJSONParsers = FormatParsers{
 	"cbor":     cborJSONParser,
 	"dag-cbor": cborJSONParser,
+
+	"protobuf": dagpbJSONParser,
+	"dag-pb":   dagpbJSONParser,
 }
 
 var defaultRawParsers = FormatParsers{
 	"cbor":     cborRawParser,
 	"dag-cbor": cborRawParser,
+
+	"protobuf": dagpbRawParser,
+	"dag-pb":   dagpbRawParser,
+}
+
+var defaultCborParsers = FormatParsers{
+	"cbor":     cborRawParser,
+	"dag-cbor": cborRawParser,
 }
 
 // ParseInputs uses DefaultInputEncParsers to parse io.Reader described by
 // input encoding and format to an instance of ipld Node
-func ParseInputs(ienc, format string, r io.Reader) ([]node.Node, error) {
-	return DefaultInputEncParsers.ParseInputs(ienc, format, r)
+func ParseInputs(ienc, format string, r io.Reader, mhType uint64, mhLen int) ([]node.Node, error) {
+	return DefaultInputEncParsers.ParseInputs(ienc, format, r, mhType, mhLen)
 }
 
 // AddParser adds DagParser under give input encoding and format
@@ -53,39 +63,16 @@ func (iep InputEncParsers) AddParser(ienv, format string, f DagParser) {
 
 // ParseInputs parses io.Reader described by input encoding and format to
 // an instance of ipld Node
-func (iep InputEncParsers) ParseInputs(ienc, format string, r io.Reader) ([]node.Node, error) {
-	pset, ok := iep[ienc]
+func (iep InputEncParsers) ParseInputs(ienc, format string, r io.Reader, mhType uint64, mhLen int) ([]node.Node, error) {
+	parsers, ok := iep[ienc]
 	if !ok {
 		return nil, fmt.Errorf("no input parser for %q", ienc)
 	}
 
-	parser, ok := pset[format]
+	parser, ok := parsers[format]
 	if !ok {
 		return nil, fmt.Errorf("no parser for format %q using input type %q", format, ienc)
 	}
 
-	return parser(r)
-}
-
-func cborJSONParser(r io.Reader) ([]node.Node, error) {
-	nd, err := ipldcbor.FromJson(r)
-	if err != nil {
-		return nil, err
-	}
-
-	return []node.Node{nd}, nil
-}
-
-func cborRawParser(r io.Reader) ([]node.Node, error) {
-	data, err := ioutil.ReadAll(r)
-	if err != nil {
-		return nil, err
-	}
-
-	nd, err := ipldcbor.Decode(data)
-	if err != nil {
-		return nil, err
-	}
-
-	return []node.Node{nd}, nil
+	return parser(r, mhType, mhLen)
 }

merkledag/merkledag.go

@@ -11,8 +11,8 @@ import (
 
 	cid "gx/ipfs/QmTprEaAA2A9bst5XH7exuyi5KzNMK3SEDNN8rBDnKWcUS/go-cid"
 	blocks "gx/ipfs/QmVA4mafxbfH5aEvNz8fyoxC6J1xhAtw88B4GerPznSZBg/go-block-format"
-	ipldcbor "gx/ipfs/QmXgUVPAxjMLZSyxx818YstJJAoRg3nyPWENmBLVzLtoax/go-ipld-cbor"
 	node "gx/ipfs/QmYNyRZJBUYPNrLszFmrBrPJbsBh2vMsefz5gnDpB5M1P6/go-ipld-format"
+	ipldcbor "gx/ipfs/QmeebqVZeEXBqJ2B4urQWfdhwRRPm84ajnCo8x8pfwbsPM/go-ipld-cbor"
 )
 
 // TODO: We should move these registrations elsewhere. Really, most of the IPLD

package.json

@@ -261,9 +261,9 @@
     },
     {
       "author": "whyrusleeping",
-      "hash": "QmXgUVPAxjMLZSyxx818YstJJAoRg3nyPWENmBLVzLtoax",
+      "hash": "QmeebqVZeEXBqJ2B4urQWfdhwRRPm84ajnCo8x8pfwbsPM",
       "name": "go-ipld-cbor",
-      "version": "1.2.7"
+      "version": "1.2.8"
     },
     {
       "author": "lgierth",

plugin/plugins/git/git.go

@@ -2,12 +2,15 @@ package git
 
 import (
 	"compress/zlib"
+	"fmt"
 	"io"
+	"math"
 
 	"github.com/ipfs/go-ipfs/core/coredag"
 	"github.com/ipfs/go-ipfs/plugin"
 
 	"gx/ipfs/QmTprEaAA2A9bst5XH7exuyi5KzNMK3SEDNN8rBDnKWcUS/go-cid"
+	mh "gx/ipfs/QmU9a9NV9RdPNwZQDYd5uKsm6N6LJLSvLbywDDYFbaaC6P/go-multihash"
 	"gx/ipfs/QmYNyRZJBUYPNrLszFmrBrPJbsBh2vMsefz5gnDpB5M1P6/go-ipld-format"
 	git "gx/ipfs/Qma7Kuwun7w8SZphjEPDVxvGfetBkqdNGmigDA13sJdLex/go-ipld-git"
 )
@@ -44,7 +47,15 @@ func (*gitPlugin) RegisterInputEncParsers(iec coredag.InputEncParsers) error {
 	return nil
 }
 
-func parseRawGit(r io.Reader) ([]format.Node, error) {
+func parseRawGit(r io.Reader, mhType uint64, mhLen int) ([]format.Node, error) {
+	if mhType != math.MaxUint64 && mhType != mh.SHA1 {
+		return nil, fmt.Errorf("unsupported mhType %d", mhType)
+	}
+
+	if mhLen != -1 && mhLen != mh.DefaultLengths[mh.SHA1] {
+		return nil, fmt.Errorf("invalid mhLen %d", mhType)
+	}
+
 	nd, err := git.ParseObject(r)
 	if err != nil {
 		return nil, err
@@ -53,12 +64,12 @@ func parseRawGit(r io.Reader) ([]format.Node, error) {
 	return []format.Node{nd}, nil
 }
 
-func parseZlibGit(r io.Reader) ([]format.Node, error) {
+func parseZlibGit(r io.Reader, mhType uint64, mhLen int) ([]format.Node, error) {
 	rc, err := zlib.NewReader(r)
 	if err != nil {
 		return nil, err
 	}
 
 	defer rc.Close()
-	return parseRawGit(rc)
+	return parseRawGit(rc, mhType, mhLen)
 }

test/sharness/t0053-dag.sh

@@ -118,6 +118,12 @@ test_dag_cmd() {
 		test_fsh echo $HASH
 	'
 
+	test_expect_success "non-canonical cbor input is normalized with input-enc cbor" '
+		HASH=$(cat ../t0053-dag-data/non-canon.cbor | ipfs dag put --format=cbor --input-enc=cbor) &&
+		test $HASH = "zdpuAmxF8q6iTUtkB3xtEYzmc5Sw762qwQJftt5iW8NTWLtjC" ||
+		test_fsh echo $HASH
+	'
+
 	test_expect_success "add an ipld with pin" '
 		PINHASH=$(printf {\"foo\":\"bar\"} | ipfs dag put --pin=true)
 	'
@@ -126,6 +132,61 @@ test_dag_cmd() {
 		ipfs repo gc > /dev/null &&
 		ipfs refs -r --timeout=2s $PINHASH > /dev/null
 	'
+
+	test_expect_success "can add an ipld object with sha3 hash" '
+		IPLDHASH=$(cat ipld_object | ipfs dag put --hash sha3)
+	'
+
+	test_expect_success "output looks correct" '
+		EXPHASH="zBwWX8u9LYZdCWqaryJW8QsBstghHSPy41nfhhFLY9qw1Vu2BWqnMFtk1jL3qCtEdGd7Kqw1HNPZv5z8LxP2eHGGDCdRE"
+		test $EXPHASH = $IPLDHASH
+	'
+
+	test_expect_success "prepare dag-pb object" '
+		echo foo > test_file &&
+		HASH=$(ipfs add -wq test_file | tail -n1)
+	'
+
+	test_expect_success "dag put with json dag-pb works" '
+		ipfs dag get $HASH > pbjson &&
+		cat pbjson | ipfs dag put --format=dag-pb --input-enc=json > dag_put_out
+	'
+
+	test_expect_success "dag put with dag-pb works output looks good" '
+		printf $HASH > dag_put_exp &&
+		test_cmp dag_put_exp dag_put_out
+	'
+
+	test_expect_success "dag put with raw dag-pb works" '
+		ipfs block get $HASH > pbraw &&
+		cat pbraw | ipfs dag put --format=dag-pb --input-enc=raw > dag_put_out
+	'
+
+	test_expect_success "dag put with dag-pb works output looks good" '
+		printf $HASH > dag_put_exp &&
+		test_cmp dag_put_exp dag_put_out
+	'
+
+	test_expect_success "prepare data for dag resolve" '
+		NESTED_HASH=$(echo "{\"data\":123}" | ipfs dag put) &&
+		HASH=$(echo "{\"obj\":{\"/\":\"${NESTED_HASH}\"}}" | ipfs dag put)
+	'
+
+	test_expect_success "dag resolve some things" '
+		ipfs dag resolve $HASH > resolve_hash &&
+		ipfs dag resolve ${HASH}/obj > resolve_obj &&
+		ipfs dag resolve ${HASH}/obj/data > resolve_data
+	'
+
+	test_expect_success "dag resolve output looks good" '
+		printf $HASH > resolve_hash_exp &&
+		printf $NESTED_HASH > resolve_obj_exp &&
+		printf $NESTED_HASH/data > resolve_data_exp &&
+
+		test_cmp resolve_hash_exp resolve_hash &&
+		test_cmp resolve_obj_exp resolve_obj &&
+		test_cmp resolve_data_exp resolve_data
+	'
 }
 
 # should work offline