Merge pull request #5331 from ipfs/refactor/coreapi/block

block cmd: Use coreapi

core/commands/block.go

 package commands
 
 import (
-	"bytes"
-	"context"
-	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 
 	util "github.com/ipfs/go-ipfs/blocks/blockstoreutil"
 	cmdenv "github.com/ipfs/go-ipfs/core/commands/cmdenv"
 	e "github.com/ipfs/go-ipfs/core/commands/e"
+	coreiface "github.com/ipfs/go-ipfs/core/coreapi/interface"
+	"github.com/ipfs/go-ipfs/core/coreapi/interface/options"
 
 	"gx/ipfs/QmPTfgFTo9PFr1PvPKyKoeMgBvYPh6cX3aDP7DHKVbnCbi/go-ipfs-cmds"
 	mh "gx/ipfs/QmPnFwZ2JXKnXgMw8CdBPxn7FWh6LLdjUjxV1fKHuJnkr8/go-multihash"
 	"gx/ipfs/QmSP88ryZkHSRn1fnngAaV2Vcn63WUJzAavnRM9CVdU1Ky/go-ipfs-cmdkit"
-	blocks "gx/ipfs/QmWAzSEoqZ6xU6pu8yL8e5WaMb7wtbfbhhN4p1DknUPtr3/go-block-format"
-	cid "gx/ipfs/QmZFbDTY9jfSBms2MchvYM9oYRbAF19K7Pby47yDBfpPrb/go-cid"
 )
 
 type BlockStat struct {
@@ -64,15 +60,27 @@ on raw IPFS blocks. It outputs the following to stdout:
 		cmdkit.StringArg("key", true, false, "The base58 multihash of an existing block to stat.").EnableStdin(),
 	},
 	Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) {
-		b, err := getBlockForKey(req.Context, env, req.Arguments[0])
+		api, err := cmdenv.GetApi(env)
+		if err != nil {
+			res.SetError(err, cmdkit.ErrNormal)
+			return
+		}
+
+		p, err := coreiface.ParsePath(req.Arguments[0])
+		if err != nil {
+			res.SetError(err, cmdkit.ErrNormal)
+			return
+		}
+
+		b, err := api.Block().Stat(req.Context, p)
 		if err != nil {
 			res.SetError(err, cmdkit.ErrNormal)
 			return
 		}
 
 		err = cmds.EmitOnce(res, &BlockStat{
-			Key:  b.Cid().String(),
-			Size: len(b.RawData()),
+			Key:  b.Path().Cid().String(),
+			Size: b.Size(),
 		})
 		if err != nil {
 			log.Error(err)
@@ -104,13 +112,25 @@ It outputs to stdout, and <key> is a base58 encoded multihash.
 		cmdkit.StringArg("key", true, false, "The base58 multihash of an existing block to get.").EnableStdin(),
 	},
 	Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) {
-		b, err := getBlockForKey(req.Context, env, req.Arguments[0])
+		api, err := cmdenv.GetApi(env)
 		if err != nil {
 			res.SetError(err, cmdkit.ErrNormal)
 			return
 		}
 
-		err = res.Emit(bytes.NewReader(b.RawData()))
+		p, err := coreiface.ParsePath(req.Arguments[0])
+		if err != nil {
+			res.SetError(err, cmdkit.ErrNormal)
+			return
+		}
+
+		r, err := api.Block().Get(req.Context, p)
+		if err != nil {
+			res.SetError(err, cmdkit.ErrNormal)
+			return
+		}
+
+		err = res.Emit(r)
 		if err != nil {
 			log.Error(err)
 		}
@@ -138,7 +158,7 @@ than 'sha2-256' or format to anything other than 'v0' will result in CIDv1.
 		cmdkit.IntOption("mhlen", "multihash hash length").WithDefault(-1),
 	},
 	Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) {
-		n, err := cmdenv.GetNode(env)
+		api, err := cmdenv.GetApi(env)
 		if err != nil {
 			res.SetError(err, cmdkit.ErrNormal)
 			return
@@ -150,18 +170,6 @@ than 'sha2-256' or format to anything other than 'v0' will result in CIDv1.
 			return
 		}
 
-		data, err := ioutil.ReadAll(file)
-		if err != nil {
-			res.SetError(err, cmdkit.ErrNormal)
-			return
-		}
-
-		err = file.Close()
-		if err != nil {
-			res.SetError(err, cmdkit.ErrNormal)
-			return
-		}
-
 		mhtype, _ := req.Options["mhtype"].(string)
 		mhtval, ok := mh.Names[mhtype]
 		if !ok {
@@ -170,62 +178,30 @@ than 'sha2-256' or format to anything other than 'v0' will result in CIDv1.
 			return
 		}
 
-		var pref cid.Prefix
-		pref.Version = 1
-
-		format, formatSet := req.Options["format"].(string)
-		if !formatSet {
-			if mhtval == mh.SHA2_256 {
-				format = "v0"
-			} else {
-				format = "protobuf"
-			}
-		}
-
-		if format == "v0" {
-			pref.Version = 0
-		}
-		formatval, ok := cid.Codecs[format]
-		if !ok {
-			res.SetError(fmt.Errorf("unrecognized format: '%s'", format), cmdkit.ErrNormal)
-			return
-		}
-		if mhtval != mh.SHA2_256 && pref.Version == 0 {
-			res.SetError(errors.New("cannot generate CIDv0 with non-sha256 hash function"), cmdkit.ErrNormal)
-			return
-		}
-
-		pref.Codec = formatval
-		pref.MhType = mhtval
-
 		mhlen, ok := req.Options["mhlen"].(int)
 		if !ok {
 			res.SetError("missing option \"mhlen\"", cmdkit.ErrNormal)
 			return
 		}
-		pref.MhLength = mhlen
 
-		bcid, err := pref.Sum(data)
-		if err != nil {
-			res.SetError(err, cmdkit.ErrNormal)
-			return
-		}
-
-		b, err := blocks.NewBlockWithCid(data, bcid)
-		if err != nil {
-			res.SetError(err, cmdkit.ErrNormal)
-			return
+		format, formatSet := req.Options["format"].(string)
+		if !formatSet {
+			if mhtval != mh.SHA2_256 || (mhlen != -1 && mhlen != 32) {
+				format = "protobuf"
+			} else {
+				format = "v0"
+			}
 		}
 
-		err = n.Blocks.AddBlock(b)
+		p, err := api.Block().Put(req.Context, file, options.Block.Hash(mhtval, mhlen), options.Block.Format(format))
 		if err != nil {
 			res.SetError(err, cmdkit.ErrNormal)
 			return
 		}
 
 		err = cmds.EmitOnce(res, &BlockStat{
-			Key:  b.Cid().String(),
-			Size: len(data),
+			Key:  p.Path().Cid().String(),
+			Size: p.Size(),
 		})
 		if err != nil {
 			log.Error(err)
@@ -244,29 +220,6 @@ than 'sha2-256' or format to anything other than 'v0' will result in CIDv1.
 	Type: BlockStat{},
 }
 
-func getBlockForKey(ctx context.Context, env cmds.Environment, skey string) (blocks.Block, error) {
-	if len(skey) == 0 {
-		return nil, fmt.Errorf("zero length cid invalid")
-	}
-
-	n, err := cmdenv.GetNode(env)
-	if err != nil {
-		return nil, err
-	}
-
-	c, err := cid.Decode(skey)
-	if err != nil {
-		return nil, err
-	}
-
-	b, err := n.Blocks.GetBlock(ctx, c)
-	if err != nil {
-		return nil, err
-	}
-
-	return b, nil
-}
-
 var blockRmCmd = &cmds.Command{
 	Helptext: cmdkit.HelpText{
 		Tagline: "Remove IPFS block(s).",
@@ -283,38 +236,42 @@ It takes a list of base58 encoded multihashes to remove.
 		cmdkit.BoolOption("quiet", "q", "Write minimal output."),
 	},
 	Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) {
-		n, err := cmdenv.GetNode(env)
+		api, err := cmdenv.GetApi(env)
 		if err != nil {
 			res.SetError(err, cmdkit.ErrNormal)
 			return
 		}
-		hashes := req.Arguments
+
 		force, _ := req.Options["force"].(bool)
 		quiet, _ := req.Options["quiet"].(bool)
-		cids := make([]*cid.Cid, 0, len(hashes))
-		for _, hash := range hashes {
-			c, err := cid.Decode(hash)
+
+		// TODO: use batching coreapi when done
+		for _, b := range req.Arguments {
+			p, err := coreiface.ParsePath(b)
 			if err != nil {
-				err = fmt.Errorf("invalid content id: %s (%s)", hash, err)
 				res.SetError(err, cmdkit.ErrNormal)
 				return
 			}
 
-			cids = append(cids, c)
-		}
-		ch, err := util.RmBlocks(n.Blockstore, n.Pinning, cids, util.RmBlocksOpts{
-			Quiet: quiet,
-			Force: force,
-		})
+			rp, err := api.ResolvePath(req.Context, p)
+			if err != nil {
+				res.SetError(err, cmdkit.ErrNormal)
+				return
+			}
 
-		if err != nil {
-			res.SetError(err, cmdkit.ErrNormal)
-			return
-		}
+			err = api.Block().Rm(req.Context, rp, options.Block.Force(force))
+			if err != nil {
+				res.Emit(&util.RemovedBlock{
+					Hash:  rp.Cid().String(),
+					Error: err.Error(),
+				})
+			}
 
-		err = res.Emit(ch)
-		if err != nil {
-			log.Error(err)
+			if !quiet {
+				res.Emit(&util.RemovedBlock{
+					Hash: rp.Cid().String(),
+				})
+			}
 		}
 	},
 	PostRun: cmds.PostRunMap{

core/coreapi/block.go

@@ -4,7 +4,6 @@ import (
 	"bytes"
 	"context"
 	"errors"
-	"fmt"
 	"io"
 	"io/ioutil"
 
@@ -12,7 +11,6 @@ import (
 	coreiface "github.com/ipfs/go-ipfs/core/coreapi/interface"
 	caopts "github.com/ipfs/go-ipfs/core/coreapi/interface/options"
 
-	mh "gx/ipfs/QmPnFwZ2JXKnXgMw8CdBPxn7FWh6LLdjUjxV1fKHuJnkr8/go-multihash"
 	blocks "gx/ipfs/QmWAzSEoqZ6xU6pu8yL8e5WaMb7wtbfbhhN4p1DknUPtr3/go-block-format"
 	cid "gx/ipfs/QmZFbDTY9jfSBms2MchvYM9oYRbAF19K7Pby47yDBfpPrb/go-cid"
 )
@@ -24,8 +22,8 @@ type BlockStat struct {
 	size int
 }
 
-func (api *BlockAPI) Put(ctx context.Context, src io.Reader, opts ...caopts.BlockPutOption) (coreiface.ResolvedPath, error) {
-	settings, err := caopts.BlockPutOptions(opts...)
+func (api *BlockAPI) Put(ctx context.Context, src io.Reader, opts ...caopts.BlockPutOption) (coreiface.BlockStat, error) {
+	_, pref, err := caopts.BlockPutOptions(opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -35,21 +33,6 @@ func (api *BlockAPI) Put(ctx context.Context, src io.Reader, opts ...caopts.Bloc
 		return nil, err
 	}
 
-	var pref cid.Prefix
-	pref.Version = 1
-
-	formatval, ok := cid.Codecs[settings.Codec]
-	if !ok {
-		return nil, fmt.Errorf("unrecognized format: %s", settings.Codec)
-	}
-	if settings.Codec == "v0" && settings.MhType == mh.SHA2_256 {
-		pref.Version = 0
-	}
-	pref.Codec = formatval
-
-	pref.MhType = settings.MhType
-	pref.MhLength = settings.MhLength
-
 	bcid, err := pref.Sum(data)
 	if err != nil {
 		return nil, err
@@ -65,7 +48,7 @@ func (api *BlockAPI) Put(ctx context.Context, src io.Reader, opts ...caopts.Bloc
 		return nil, err
 	}
 
-	return coreiface.IpldPath(b.Cid()), nil
+	return &BlockStat{path: coreiface.IpldPath(b.Cid()), size: len(data)}, nil
 }
 
 func (api *BlockAPI) Get(ctx context.Context, p coreiface.Path) (io.Reader, error) {

core/coreapi/block_test.go

@@ -6,6 +6,7 @@ import (
 	"strings"
 	"testing"
 
+	coreiface "github.com/ipfs/go-ipfs/core/coreapi/interface"
 	opt "github.com/ipfs/go-ipfs/core/coreapi/interface/options"
 
 	mh "gx/ipfs/QmPnFwZ2JXKnXgMw8CdBPxn7FWh6LLdjUjxV1fKHuJnkr8/go-multihash"
@@ -23,8 +24,8 @@ func TestBlockPut(t *testing.T) {
 		t.Error(err)
 	}
 
-	if res.Cid().String() != "QmPyo15ynbVrSTVdJL9th7JysHaAbXt9dM9tXk1bMHbRtk" {
-		t.Errorf("got wrong cid: %s", res.Cid().String())
+	if res.Path().Cid().String() != "QmPyo15ynbVrSTVdJL9th7JysHaAbXt9dM9tXk1bMHbRtk" {
+		t.Errorf("got wrong cid: %s", res.Path().Cid().String())
 	}
 }
 
@@ -40,8 +41,8 @@ func TestBlockPutFormat(t *testing.T) {
 		t.Error(err)
 	}
 
-	if res.Cid().String() != "zdpuAn4amuLWo8Widi5v6VQpuo2dnpnwbVE3oB6qqs7mDSeoa" {
-		t.Errorf("got wrong cid: %s", res.Cid().String())
+	if res.Path().Cid().String() != "zdpuAn4amuLWo8Widi5v6VQpuo2dnpnwbVE3oB6qqs7mDSeoa" {
+		t.Errorf("got wrong cid: %s", res.Path().Cid().String())
 	}
 }
 
@@ -54,11 +55,11 @@ func TestBlockPutHash(t *testing.T) {
 
 	res, err := api.Block().Put(ctx, strings.NewReader(`Hello`), opt.Block.Hash(mh.KECCAK_512, -1))
 	if err != nil {
-		t.Error(err)
+		t.Fatal(err)
 	}
 
-	if res.Cid().String() != "zBurKB9YZkcDf6xa53WBE8CFX4ydVqAyf9KPXBFZt5stJzEstaS8Hukkhu4gwpMtc1xHNDbzP7sPtQKyWsP3C8fbhkmrZ" {
-		t.Errorf("got wrong cid: %s", res.Cid().String())
+	if res.Path().Cid().String() != "zBurKB9YZkcDf6xa53WBE8CFX4ydVqAyf9KPXBFZt5stJzEstaS8Hukkhu4gwpMtc1xHNDbzP7sPtQKyWsP3C8fbhkmrZ" {
+		t.Errorf("got wrong cid: %s", res.Path().Cid().String())
 	}
 }
 
@@ -74,7 +75,7 @@ func TestBlockGet(t *testing.T) {
 		t.Error(err)
 	}
 
-	r, err := api.Block().Get(ctx, res)
+	r, err := api.Block().Get(ctx, res.Path())
 	if err != nil {
 		t.Error(err)
 	}
@@ -87,6 +88,19 @@ func TestBlockGet(t *testing.T) {
 	if string(d) != "Hello" {
 		t.Error("didn't get correct data back")
 	}
+
+	p, err := coreiface.ParsePath("/ipfs/" + res.Path().Cid().String())
+	if err != nil {
+		t.Error(err)
+	}
+
+	rp, err := api.ResolvePath(ctx, p)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if rp.Cid().String() != res.Path().Cid().String() {
+		t.Error("paths didn't match")
+	}
 }
 
 func TestBlockRm(t *testing.T) {
@@ -101,7 +115,7 @@ func TestBlockRm(t *testing.T) {
 		t.Error(err)
 	}
 
-	r, err := api.Block().Get(ctx, res)
+	r, err := api.Block().Get(ctx, res.Path())
 	if err != nil {
 		t.Error(err)
 	}
@@ -115,12 +129,12 @@ func TestBlockRm(t *testing.T) {
 		t.Error("didn't get correct data back")
 	}
 
-	err = api.Block().Rm(ctx, res)
+	err = api.Block().Rm(ctx, res.Path())
 	if err != nil {
 		t.Error(err)
 	}
 
-	_, err = api.Block().Get(ctx, res)
+	_, err = api.Block().Get(ctx, res.Path())
 	if err == nil {
 		t.Error("expected err to exist")
 	}
@@ -128,7 +142,7 @@ func TestBlockRm(t *testing.T) {
 		t.Errorf("unexpected error; %s", err.Error())
 	}
 
-	err = api.Block().Rm(ctx, res)
+	err = api.Block().Rm(ctx, res.Path())
 	if err == nil {
 		t.Error("expected err to exist")
 	}
@@ -136,7 +150,7 @@ func TestBlockRm(t *testing.T) {
 		t.Errorf("unexpected error; %s", err.Error())
 	}
 
-	err = api.Block().Rm(ctx, res, opt.Block.Force(true))
+	err = api.Block().Rm(ctx, res.Path(), opt.Block.Force(true))
 	if err != nil {
 		t.Error(err)
 	}
@@ -154,12 +168,12 @@ func TestBlockStat(t *testing.T) {
 		t.Error(err)
 	}
 
-	stat, err := api.Block().Stat(ctx, res)
+	stat, err := api.Block().Stat(ctx, res.Path())
 	if err != nil {
 		t.Error(err)
 	}
 
-	if stat.Path().String() != res.String() {
+	if stat.Path().String() != res.Path().String() {
 		t.Error("paths don't match")
 	}
 

core/coreapi/interface/block.go

@@ -19,7 +19,7 @@ type BlockStat interface {
 // BlockAPI specifies the interface to the block layer
 type BlockAPI interface {
 	// Put imports raw block data, hashing it using specified settings.
-	Put(context.Context, io.Reader, ...options.BlockPutOption) (ResolvedPath, error)
+	Put(context.Context, io.Reader, ...options.BlockPutOption) (BlockStat, error)
 
 	// Get attempts to resolve the path and return a reader for data in the block
 	Get(context.Context, Path) (io.Reader, error)

core/coreapi/interface/options/block.go

 package options
 
 import (
-	"gx/ipfs/QmPnFwZ2JXKnXgMw8CdBPxn7FWh6LLdjUjxV1fKHuJnkr8/go-multihash"
+	"fmt"
+	mh "gx/ipfs/QmPnFwZ2JXKnXgMw8CdBPxn7FWh6LLdjUjxV1fKHuJnkr8/go-multihash"
+	cid "gx/ipfs/QmZFbDTY9jfSBms2MchvYM9oYRbAF19K7Pby47yDBfpPrb/go-cid"
 )
 
 type BlockPutSettings struct {
@@ -17,20 +19,52 @@ type BlockRmSettings struct {
 type BlockPutOption func(*BlockPutSettings) error
 type BlockRmOption func(*BlockRmSettings) error
 
-func BlockPutOptions(opts ...BlockPutOption) (*BlockPutSettings, error) {
+func BlockPutOptions(opts ...BlockPutOption) (*BlockPutSettings, cid.Prefix, error) {
 	options := &BlockPutSettings{
-		Codec:    "v0",
-		MhType:   multihash.SHA2_256,
+		Codec:    "",
+		MhType:   mh.SHA2_256,
 		MhLength: -1,
 	}
 
 	for _, opt := range opts {
 		err := opt(options)
 		if err != nil {
-			return nil, err
+			return nil, cid.Prefix{}, err
 		}
 	}
-	return options, nil
+
+	var pref cid.Prefix
+	pref.Version = 1
+
+	if options.Codec == "" {
+		if options.MhType != mh.SHA2_256 || (options.MhLength != -1 && options.MhLength != 32) {
+			options.Codec = "protobuf"
+		} else {
+			options.Codec = "v0"
+		}
+	}
+
+	if options.Codec == "v0" && options.MhType == mh.SHA2_256 {
+		pref.Version = 0
+	}
+
+	formatval, ok := cid.Codecs[options.Codec]
+	if !ok {
+		return nil, cid.Prefix{}, fmt.Errorf("unrecognized format: %s", options.Codec)
+	}
+
+	if options.Codec == "v0" {
+		if options.MhType != mh.SHA2_256 || (options.MhLength != -1 && options.MhLength != 32) {
+			return nil, cid.Prefix{}, fmt.Errorf("only sha2-255-32 is allowed with CIDv0")
+		}
+	}
+
+	pref.Codec = formatval
+
+	pref.MhType = options.MhType
+	pref.MhLength = options.MhLength
+
+	return options, pref, nil
 }
 
 func BlockRmOptions(opts ...BlockRmOption) (*BlockRmSettings, error) {

core/coreapi/path_test.go

@@ -31,7 +31,7 @@ func TestMutablePath(t *testing.T) {
 		t.Error(err)
 	}
 
-	if blk.Mutable() {
+	if blk.Path().Mutable() {
 		t.Error("expected /ipld path to be immutable")
 	}
 }
@@ -129,7 +129,7 @@ func TestPathRoot(t *testing.T) {
 		t.Error(err)
 	}
 
-	obj, err := api.Dag().Put(ctx, strings.NewReader(`{"foo": {"/": "`+blk.Cid().String()+`"}}`))
+	obj, err := api.Dag().Put(ctx, strings.NewReader(`{"foo": {"/": "`+blk.Path().Cid().String()+`"}}`))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -148,7 +148,7 @@ func TestPathRoot(t *testing.T) {
 		t.Error("unexpected path root")
 	}
 
-	if rp.Cid().String() != blk.Cid().String() {
+	if rp.Cid().String() != blk.Path().Cid().String() {
 		t.Error("unexpected path cid")
 	}
 }