filestore util: Add basic implementation of 'filestore get' command.

License: MIT
Signed-off-by: Kevin Atkinson <k@kevina.org>

core/commands/filestore.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"path/filepath"
 
 	oldCmds "github.com/ipfs/go-ipfs/commands"
 	"github.com/ipfs/go-ipfs/core"
@@ -26,6 +27,7 @@ var FileStoreCmd = &cmds.Command{
 	OldSubcommands: map[string]*oldCmds.Command{
 		"verify": verifyFileStore,
 		"dups":   dupsFileStore,
+		"get":    getFileStoreCmd,
 	},
 }
 
@@ -236,6 +238,52 @@ var dupsFileStore = &oldCmds.Command{
 	Type:       RefWrapper{},
 }
 
+var getFileStoreCmd = &oldCmds.Command{
+	Helptext: cmdkit.HelpText{
+		Tagline: "Download IPFS objects into filestore.",
+	},
+	Arguments: []cmdkit.Argument{
+		cmdkit.StringArg("cid", true, false, "The cid of the IPFS object to be outputted."),
+		cmdkit.StringArg("file-path", true, false, "Path of file to store object in."),
+	},
+	Run: func(req oldCmds.Request, res oldCmds.Response) {
+		node, fs, err := getFilestore(req.InvocContext())
+		ctx := req.Context()
+		if err != nil {
+			res.SetError(err, cmdkit.ErrNormal)
+			return
+		}
+		args := req.Arguments()
+		c, err := cid.Decode(args[0])
+		if err != nil {
+			res.SetError(err, cmdkit.ErrNormal)
+			return
+		}
+		filePath, err := filepath.Abs(args[1])
+		if err != nil {
+			res.SetError(err, cmdkit.ErrNormal)
+			return
+		}
+		g := &filestore.Getter{
+			Ctx:       ctx,
+			FullPath:  filePath,
+			Nodes:     node.DAG,
+			Filestore: fs,
+		}
+		err = g.Init()
+		if err != nil {
+			res.SetError(err, cmdkit.ErrNormal)
+			return
+		}
+		_, err = g.Get(c, 0)
+		if err != nil {
+			res.SetError(err, cmdkit.ErrNormal)
+			return
+		}
+		res.SetOutput(nil)
+	},
+}
+
 type getNoder interface {
 	GetNode() (*core.IpfsNode, error)
 }

filestore/fsrefstore.go

@@ -206,11 +206,21 @@ func (f *FileManager) Put(b *posinfo.FilestoreNode) error {
 	return f.putTo(b, f.ds)
 }
 
+// CheckPath checks that a path to the backing file is valid for use
+// in the filestore
+func (f *FileManager) CheckPath(fullpath string) error {
+	if !filepath.HasPrefix(fullpath, f.root) {
+		return fmt.Errorf("cannot add filestore references outside ipfs root (%s)", f.root)
+	}
+	return nil
+}
+
 func (f *FileManager) putTo(b *posinfo.FilestoreNode, to putter) error {
 	var dobj pb.DataObj
 
-	if !filepath.HasPrefix(b.PosInfo.FullPath, f.root) {
-		return fmt.Errorf("cannot add filestore references outside ipfs root (%s)", f.root)
+	err := f.CheckPath(b.PosInfo.FullPath)
+	if err != nil {
+		return err
 	}
 
 	p, err := filepath.Rel(f.root, b.PosInfo.FullPath)

filestore/get.go

+package filestore
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	dag "github.com/ipfs/go-ipfs/merkledag"
+	pi "github.com/ipfs/go-ipfs/thirdparty/posinfo"
+	unixfs "github.com/ipfs/go-ipfs/unixfs"
+
+	cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid"
+	node "gx/ipfs/QmPN7cwmpcc4DWXb4KTB9dNAJgjuPY69h3npsMfhRrQL9c/go-ipld-format"
+)
+
+// Getter gets nodes directly into the filestore.  Call Init before
+// the first use and then Get to get a node.
+type Getter struct {
+	Ctx       context.Context
+	FullPath  string
+	Nodes     node.NodeGetter
+	Filestore *Filestore
+	fh        *os.File
+}
+
+// Init inits the filestore getter
+func (g *Getter) Init() error {
+	err := g.Filestore.FileManager().CheckPath(g.FullPath)
+	if err != nil {
+		return err
+	}
+
+	g.fh, err = os.Create(g.FullPath)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// Get gets a node directly into the filestore
+func (g *Getter) Get(c *cid.Cid, offset uint64) (uint64, error) {
+	node, err := g.Nodes.Get(g.Ctx, c)
+	if err != nil {
+		return 0, err
+	}
+	switch n := node.(type) {
+	case *dag.ProtoNode:
+		pbn, err := unixfs.FromBytes(n.Data())
+		if err != nil {
+			return 0, err
+		}
+		if len(pbn.Data) != 0 {
+			return 0, fmt.Errorf("%s: unsupported node type", c.String())
+		}
+		// still need to store the node, incase the node getter
+		// bypasses the normal blockstore
+		err = g.Filestore.Put(n)
+		if err != nil {
+			return 0, err
+		}
+		for _, lnk := range n.Links() {
+			offset, err = g.Get(lnk.Cid, offset)
+			if err != nil {
+				return 0, err
+			}
+		}
+		return offset, nil
+	case *dag.RawNode:
+		data := n.RawData()
+		_, err := g.fh.WriteAt(data, int64(offset))
+		if err != nil {
+			return 0, err
+		}
+		fsn := &pi.FilestoreNode{node, &pi.PosInfo{
+			Offset:   offset,
+			FullPath: g.FullPath,
+		}}
+		err = g.Filestore.Put(fsn)
+		if err != nil {
+			return 0, err
+		}
+		return offset + uint64(len(data)), nil
+	default:
+		return 0, fmt.Errorf("%s: unsupported node type", c.String())
+	}
+
+}

test/sharness/t0271-filestore-utils.sh

@@ -165,6 +165,25 @@ test_filestore_dups() {
   '
 }
 
+test_filestore_get() {
+  test_expect_success "create and add some files" '
+    random    1000  11 > smallfile0 &&
+    random 1000000  12 > largefile0 &&
+    SMALLHASH=$(ipfs add -q --raw-leaves smallfile0) &&
+    LARGEHASH=$(ipfs add -q --raw-leaves largefile0)
+  '
+
+  test_expect_success "get small file into filestore" '
+    ipfs filestore get $SMALLHASH smallfile &&
+    test_cmp smallfile0 smallfile
+  '
+
+  test_expect_success "get large file into filestore" '
+    ipfs filestore get $LARGEHASH largefile &&
+    test_cmp largefile0 largefile
+  '
+}
+
 #
 # No daemon
 #
@@ -177,6 +196,8 @@ test_filestore_verify
 
 test_filestore_dups
 
+test_filestore_get
+
 #
 # With daemon
 #
@@ -193,6 +214,8 @@ test_filestore_verify
 
 test_filestore_dups
 
+test_filestore_get
+
 test_kill_ipfs_daemon
 
 test_done