filestore util: allow listing/verifying of individual blocks.

License: MIT
Signed-off-by: Kevin Atkinson <k@kevina.org>

core/commands/filestore.go

@@ -8,6 +8,7 @@ import (
 	"github.com/ipfs/go-ipfs/core"
 	"github.com/ipfs/go-ipfs/filestore"
 	u "gx/ipfs/QmZuY8aV7zbNXVy6DyN9SmnuH3o9nG852F4aTiSBpts8d1/go-ipfs-util"
+	cid "gx/ipfs/QmV5gPoRsjN1Gid3LMdNZTyfCtP2DsvqEbMAmz82RmmiGk/go-cid"
 )
 
 var FileStoreCmd = &cmds.Command{
@@ -24,19 +25,30 @@ var lsFileStore = &cmds.Command{
 	Helptext: cmds.HelpText{
 		Tagline: "List objects in filestore.",
 	},
+	Arguments: []cmds.Argument{
+		cmds.StringArg("obj", false, true, "Cid of objects to list."),
+	},
 	Run: func(req cmds.Request, res cmds.Response) {
 		_, fs, err := getFilestore(req)
 		if err != nil {
 			res.SetError(err, cmds.ErrNormal)
 			return
 		}
-		next, err := filestore.ListAll(fs)
-		if err != nil {
-			res.SetError(err, cmds.ErrNormal)
-			return
+		args := req.Arguments()
+		if len(args) > 0 {
+			out := perKeyActionToChan(args, func(c *cid.Cid) *filestore.ListRes {
+				return filestore.List(fs, c)
+			}, req.Context())
+			res.SetOutput(out)
+		} else {
+			next, err := filestore.ListAll(fs)
+			if err != nil {
+				res.SetError(err, cmds.ErrNormal)
+				return
+			}
+			out := listResToChan(next, req.Context())
+			res.SetOutput(out)
 		}
-		out := listResAsChan(next, req.Context())
-		res.SetOutput(out)
 	},
 	PostRun: func(req cmds.Request, res cmds.Response) {
 		if res.Error() != nil {
@@ -52,6 +64,7 @@ var lsFileStore = &cmds.Command{
 		for r0 := range outChan {
 			r := r0.(*filestore.ListRes)
 			if r.ErrorMsg != "" {
+				errors = true
 				fmt.Fprintf(res.Stderr(), "%s\n", r.ErrorMsg)
 			} else {
 				fmt.Fprintf(res.Stdout(), "%s\n", r.FormatLong())
@@ -68,19 +81,30 @@ var verifyFileStore = &cmds.Command{
 	Helptext: cmds.HelpText{
 		Tagline: "Verify objects in filestore.",
 	},
+	Arguments: []cmds.Argument{
+		cmds.StringArg("obj", false, true, "Cid of objects to verify."),
+	},
 	Run: func(req cmds.Request, res cmds.Response) {
 		_, fs, err := getFilestore(req)
 		if err != nil {
 			res.SetError(err, cmds.ErrNormal)
 			return
 		}
-		next, err := filestore.VerifyAll(fs)
-		if err != nil {
-			res.SetError(err, cmds.ErrNormal)
-			return
+		args := req.Arguments()
+		if len(args) > 0 {
+			out := perKeyActionToChan(args, func(c *cid.Cid) *filestore.ListRes {
+				return filestore.Verify(fs, c)
+			}, req.Context())
+			res.SetOutput(out)
+		} else {
+			next, err := filestore.VerifyAll(fs)
+			if err != nil {
+				res.SetError(err, cmds.ErrNormal)
+				return
+			}
+			out := listResToChan(next, req.Context())
+			res.SetOutput(out)
 		}
-		out := listResAsChan(next, req.Context())
-		res.SetOutput(out)
 	},
 	PostRun: func(req cmds.Request, res cmds.Response) {
 		if res.Error() != nil {
@@ -94,6 +118,9 @@ var verifyFileStore = &cmds.Command{
 		res.SetOutput(nil)
 		for r0 := range outChan {
 			r := r0.(*filestore.ListRes)
+			if r.Status == filestore.StatusOtherError {
+				fmt.Fprintf(res.Stderr(), "%s\n", r.ErrorMsg)
+			}
 			fmt.Fprintf(res.Stdout(), "%s %s\n", r.Status.Format(), r.FormatLong())
 		}
 	},
@@ -112,7 +139,7 @@ func getFilestore(req cmds.Request) (*core.IpfsNode, *filestore.Filestore, error
 	return n, fs, err
 }
 
-func listResAsChan(next func() *filestore.ListRes, ctx context.Context) <-chan interface{} {
+func listResToChan(next func() *filestore.ListRes, ctx context.Context) <-chan interface{} {
 	out := make(chan interface{}, 128)
 	go func() {
 		defer close(out)
@@ -130,3 +157,27 @@ func listResAsChan(next func() *filestore.ListRes, ctx context.Context) <-chan i
 	}()
 	return out
 }
+
+func perKeyActionToChan(args []string, action func(*cid.Cid) *filestore.ListRes, ctx context.Context) <-chan interface{} {
+	out := make(chan interface{}, 128)
+	go func() {
+		defer close(out)
+		for _, arg := range args {
+			c, err := cid.Decode(arg)
+			if err != nil {
+				out <- &filestore.ListRes{
+					Status:   filestore.StatusOtherError,
+					ErrorMsg: fmt.Sprintf("%s: %v", arg, err),
+				}
+				continue
+			}
+			r := action(c)
+			select {
+			case out <- r:
+			case <-ctx.Done():
+				return
+			}
+		}
+	}()
+	return out
+}

filestore/util.go

@@ -3,6 +3,7 @@ package filestore
 import (
 	"fmt"
 
+	"github.com/ipfs/go-ipfs/blocks/blockstore"
 	pb "github.com/ipfs/go-ipfs/filestore/pb"
 	dshelp "github.com/ipfs/go-ipfs/thirdparty/ds-help"
 
@@ -19,7 +20,8 @@ const (
 	StatusFileError Status = 10 // Backing File Error
 	//StatusFileNotFound Status = 11 // Backing File Not Found
 	//StatusFileChanged  Status = 12 // Contents of the file changed
-	StatusOtherError Status = 20 // Internal Error, likely corrupt entry
+	StatusOtherError  Status = 20 // Internal Error, likely corrupt entry
+	StatusKeyNotFound Status = 30
 )
 
 func (s Status) String() string {
@@ -30,13 +32,15 @@ func (s Status) String() string {
 		return "error"
 	case StatusOtherError:
 		return "ERROR"
+	case StatusKeyNotFound:
+		return "missing"
 	default:
 		return "???"
 	}
 }
 
 func (s Status) Format() string {
-	return fmt.Sprintf("%-5s", s.String())
+	return fmt.Sprintf("%-7s", s.String())
 }
 
 type ListRes struct {
@@ -52,19 +56,40 @@ func (r *ListRes) FormatLong() string {
 	switch {
 	case r.Key == nil:
 		return "?????????????????????????????????????????????????"
+	case r.FilePath == "":
+		return r.Key.String()
 	default:
 		return fmt.Sprintf("%-50s %6d %s %d", r.Key, r.Size, r.FilePath, r.Offset)
 	}
 }
 
+func List(fs *Filestore, key *cid.Cid) *ListRes {
+	return list(fs, false, key)
+}
+
 func ListAll(fs *Filestore) (func() *ListRes, error) {
 	return listAll(fs, false)
 }
 
+func Verify(fs *Filestore, key *cid.Cid) *ListRes {
+	return list(fs, true, key)
+}
+
 func VerifyAll(fs *Filestore) (func() *ListRes, error) {
 	return listAll(fs, true)
 }
 
+func list(fs *Filestore, verify bool, key *cid.Cid) *ListRes {
+	dobj, err := fs.fm.getDataObj(key)
+	if err != nil {
+		return mkListRes(key, nil, err)
+	}
+	if verify {
+		_, err = fs.fm.readDataObj(key, dobj)
+	}
+	return mkListRes(key, dobj, err)
+}
+
 func listAll(fs *Filestore, verify bool) (func() *ListRes, error) {
 	q := dsq.Query{}
 	qr, err := fs.fm.ds.Query(q)
@@ -112,7 +137,9 @@ func mkListRes(c *cid.Cid, d *pb.DataObj, err error) *ListRes {
 	status := StatusOk
 	errorMsg := ""
 	if err != nil {
-		if _, ok := err.(*CorruptReferenceError); ok {
+		if err == ds.ErrNotFound || err == blockstore.ErrNotFound {
+			status = StatusKeyNotFound
+		} else if _, ok := err.(*CorruptReferenceError); ok {
 			status = StatusFileError
 		} else {
 			status = StatusOtherError