filestore util: Add 'filestore dups' command. Enhance tests.

License: MIT
Signed-off-by: Kevin Atkinson <k@kevina.org>

core/commands/filestore.go

@@ -18,6 +18,7 @@ var FileStoreCmd = &cmds.Command{
 	Subcommands: map[string]*cmds.Command{
 		"ls":     lsFileStore,
 		"verify": verifyFileStore,
+		"dups":   dupsFileStore,
 	},
 }
 
@@ -157,6 +158,44 @@ For ERROR entries the error will also be printed to stderr.
 	Type: filestore.ListRes{},
 }
 
+var dupsFileStore = &cmds.Command{
+	Helptext: cmds.HelpText{
+		Tagline: "Print block both in filestore and non-filestore.",
+	},
+	Run: func(req cmds.Request, res cmds.Response) {
+		_, fs, err := getFilestore(req)
+		if err != nil {
+			res.SetError(err, cmds.ErrNormal)
+			return
+		}
+		ch, err := fs.FileManager().AllKeysChan(req.Context())
+		if err != nil {
+			res.SetError(err, cmds.ErrNormal)
+			return
+		}
+
+		out := make(chan interface{}, 128)
+		res.SetOutput((<-chan interface{})(out))
+
+		go func() {
+			defer close(out)
+			for cid := range ch {
+				have, err := fs.MainBlockstore().Has(cid)
+				if err != nil {
+					out <- &RefWrapper{Err: err.Error()}
+					return
+				}
+				if have {
+					out <- &RefWrapper{Ref: cid.String()}
+				}
+			}
+		}()
+	},
+	Marshalers: refsMarshallerMap,
+	Type:       RefWrapper{},
+}
+
+
 func getFilestore(req cmds.Request) (*core.IpfsNode, *filestore.Filestore, error) {
 	n, err := req.InvocContext().GetNode()
 	if err != nil {
@@ -211,3 +250,4 @@ func perKeyActionToChan(args []string, action func(*cid.Cid) *filestore.ListRes,
 	}()
 	return out
 }
+

filestore/filestore.go

@@ -19,6 +19,14 @@ type Filestore struct {
 	bs blockstore.Blockstore
 }
 
+func (f *Filestore) FileManager() *FileManager {
+	return f.fm
+}
+
+func (f *Filestore) MainBlockstore() blockstore.Blockstore {
+	return f.bs
+}
+
 func NewFilestore(bs blockstore.Blockstore, fm *FileManager) *Filestore {
 	return &Filestore{fm, bs}
 }

test/sharness/t0271-filestore-verify.sh → test/sharness/t0271-filestore-utils.sh

@@ -115,6 +115,18 @@ test_filestore_verify() {
 	'
 }
 
+cat <<EOF > dups_expect
+$FILE1_HASH
+EOF
+
+test_filestore_dups() {
+	test_expect_success "'ipfs filestore dups'" '
+		ipfs add --raw-leaves somedir/file1 &&
+		ipfs filestore dups > dups_actual &&
+		test_cmp dups_expect dups_actual
+'
+}
+
 init_ipfs_filestore() {
 	test_expect_success "clean up old node" '
 		rm -rf "$IPFS_PATH" mountdir ipfs ipns
@@ -135,6 +147,8 @@ test_filestore_adds
 
 test_filestore_verify
 
+test_filestore_dups
+
 echo "WORKING DIR"
 echo "IPFS PATH = " $IPFS_PATH
 pwd
@@ -144,10 +158,16 @@ test_init_dataset
 
 init_ipfs_filestore
 
-test_launch_ipfs_daemon
+# must be in offline mode so tests of retrieving non-exist blocks
+# don't hang
+test_launch_ipfs_daemon --offline
 
 test_filestore_adds
 
+test_filestore_verify
+
+test_filestore_dups
+
 test_kill_ipfs_daemon
 
 test_done