Merge pull request #2962 from ipfs/kevina/block-rm

Add "ipfs block rm" command.

blocks/blockstore/arc_cache.go

@@ -32,7 +32,7 @@ func (b *arccache) DeleteBlock(k key.Key) error {
 	switch err {
 	case nil, ds.ErrNotFound, ErrNotFound:
 		b.arc.Add(k, false)
-		return nil
+		return err
 	default:
 		return err
 	}

core/commands/block.go

@@ -9,8 +9,11 @@ import (
 	"strings"
 
 	"github.com/ipfs/go-ipfs/blocks"
+	bs "github.com/ipfs/go-ipfs/blocks/blockstore"
 	key "github.com/ipfs/go-ipfs/blocks/key"
 	cmds "github.com/ipfs/go-ipfs/commands"
+	"github.com/ipfs/go-ipfs/pin"
+	ds "gx/ipfs/QmTxLSvdhwg68WJimdS6icLPhZi28aTp6b7uihC2Yb47Xk/go-datastore"
 	mh "gx/ipfs/QmYf7ng2hG5XBtJA3tN34DQ2GUN5HNksEw1rLDkmr6vGku/go-multihash"
 	u "gx/ipfs/QmZNVWh8LLjAavuQ2JXuFmuYH3C11xo988vSgp7UQrTRj1/go-ipfs-util"
 )
@@ -38,6 +41,7 @@ multihash.
 		"stat": blockStatCmd,
 		"get":  blockGetCmd,
 		"put":  blockPutCmd,
+		"rm":   blockRmCmd,
 	},
 }
 
@@ -185,3 +189,135 @@ func getBlockForKey(req cmds.Request, skey string) (blocks.Block, error) {
 	log.Debugf("ipfs block: got block with key: %q", b.Key())
 	return b, nil
 }
+
+var blockRmCmd = &cmds.Command{
+	Helptext: cmds.HelpText{
+		Tagline: "Remove IPFS block(s).",
+		ShortDescription: `
+'ipfs block rm' is a plumbing command for removing raw ipfs blocks.
+It takes a list of base58 encoded multihashs to remove.
+`,
+	},
+	Arguments: []cmds.Argument{
+		cmds.StringArg("hash", true, true, "Bash58 encoded multihash of block(s) to remove."),
+	},
+	Options: []cmds.Option{
+		cmds.BoolOption("force", "f", "Ignore nonexistent blocks.").Default(false),
+		cmds.BoolOption("quiet", "q", "Write minimal output.").Default(false),
+	},
+	Run: func(req cmds.Request, res cmds.Response) {
+		n, err := req.InvocContext().GetNode()
+		if err != nil {
+			res.SetError(err, cmds.ErrNormal)
+			return
+		}
+		hashes := req.Arguments()
+		force, _, _ := req.Option("force").Bool()
+		quiet, _, _ := req.Option("quiet").Bool()
+		keys := make([]key.Key, 0, len(hashes))
+		for _, hash := range hashes {
+			k := key.B58KeyDecode(hash)
+			keys = append(keys, k)
+		}
+		outChan := make(chan interface{})
+		res.SetOutput((<-chan interface{})(outChan))
+		go func() {
+			defer close(outChan)
+			pinning := n.Pinning
+			err := rmBlocks(n.Blockstore, pinning, outChan, keys, rmBlocksOpts{
+				quiet: quiet,
+				force: force,
+			})
+			if err != nil {
+				outChan <- &RemovedBlock{Error: err.Error()}
+			}
+		}()
+		return
+	},
+	PostRun: func(req cmds.Request, res cmds.Response) {
+		if res.Error() != nil {
+			return
+		}
+		outChan, ok := res.Output().(<-chan interface{})
+		if !ok {
+			res.SetError(u.ErrCast(), cmds.ErrNormal)
+			return
+		}
+		res.SetOutput(nil)
+
+		someFailed := false
+		for out := range outChan {
+			o := out.(*RemovedBlock)
+			if o.Hash == "" && o.Error != "" {
+				res.SetError(fmt.Errorf("aborted: %s", o.Error), cmds.ErrNormal)
+				return
+			} else if o.Error != "" {
+				someFailed = true
+				fmt.Fprintf(res.Stderr(), "cannot remove %s: %s\n", o.Hash, o.Error)
+			} else {
+				fmt.Fprintf(res.Stdout(), "removed %s\n", o.Hash)
+			}
+		}
+		if someFailed {
+			res.SetError(fmt.Errorf("some blocks not removed"), cmds.ErrNormal)
+		}
+	},
+	Type: RemovedBlock{},
+}
+
+type RemovedBlock struct {
+	Hash  string `json:",omitempty"`
+	Error string `json:",omitempty"`
+}
+
+type rmBlocksOpts struct {
+	quiet bool
+	force bool	
+}
+
+func rmBlocks(blocks bs.GCBlockstore, pins pin.Pinner, out chan<- interface{}, keys []key.Key, opts rmBlocksOpts) error {
+	unlocker := blocks.GCLock()
+	defer unlocker.Unlock()
+
+	stillOkay, err := checkIfPinned(pins, keys, out)
+	if err != nil {
+		return fmt.Errorf("pin check failed: %s", err)
+	}
+
+	for _, k := range stillOkay {
+		err := blocks.DeleteBlock(k)
+		if err != nil && opts.force && (err == bs.ErrNotFound || err == ds.ErrNotFound) {
+			// ignore non-existent blocks
+		} else if err != nil {
+			out <- &RemovedBlock{Hash: k.String(), Error: err.Error()}
+		} else if !opts.quiet {
+			out <- &RemovedBlock{Hash: k.String()}
+		}
+	}
+	return nil
+}
+
+func checkIfPinned(pins pin.Pinner, keys []key.Key, out chan<- interface{}) ([]key.Key, error) {
+	stillOkay := make([]key.Key, 0, len(keys))
+	res, err := pins.CheckIfPinned(keys...)
+	if err != nil {
+		return nil, err
+	}
+	for _, r := range res {
+		switch r.Mode {
+		case pin.NotPinned:
+			stillOkay = append(stillOkay, r.Key)
+		case pin.Indirect:
+			out <- &RemovedBlock{
+				Hash:  r.Key.String(),
+				Error: fmt.Sprintf("pinned via %s", r.Via)}
+		default:
+			modeStr, _ := pin.PinModeToString(r.Mode)
+			out <- &RemovedBlock{
+				Hash:  r.Key.String(),
+				Error: fmt.Sprintf("pinned: %s", modeStr)}
+
+		}
+	}
+	return stillOkay, nil
+}

pin/pin.go

@@ -75,6 +75,10 @@ type Pinner interface {
 	Pin(context.Context, *mdag.Node, bool) error
 	Unpin(context.Context, key.Key, bool) error
 
+	// Check if a set of keys are pinned, more efficient than
+	// calling IsPinned for each key
+	CheckIfPinned(keys ...key.Key) ([]Pinned, error)
+
 	// PinWithMode is for manually editing the pin structure. Use with
 	// care! If used improperly, garbage collection may not be
 	// successful.
@@ -90,6 +94,12 @@ type Pinner interface {
 	InternalPins() []key.Key
 }
 
+type Pinned struct {
+	Key  key.Key
+	Mode PinMode
+	Via  key.Key
+}
+
 // pinner implements the Pinner interface
 type pinner struct {
 	lock       sync.RWMutex
@@ -255,6 +265,70 @@ func (p *pinner) isPinnedWithType(k key.Key, mode PinMode) (string, bool, error)
 	return "", false, nil
 }
 
+func (p *pinner) CheckIfPinned(keys ...key.Key) ([]Pinned, error) {
+	p.lock.RLock()
+	defer p.lock.RUnlock()
+	pinned := make([]Pinned, 0, len(keys))
+	toCheck := make(map[key.Key]struct{})
+
+	// First check for non-Indirect pins directly
+	for _, k := range keys {
+		if p.recursePin.HasKey(k) {
+			pinned = append(pinned, Pinned{Key: k, Mode: Recursive})
+		} else if p.directPin.HasKey(k) {
+			pinned = append(pinned, Pinned{Key: k, Mode: Direct})
+		} else if p.isInternalPin(k) {
+			pinned = append(pinned, Pinned{Key: k, Mode: Internal})
+		} else {
+			toCheck[k] = struct{}{}
+		}
+	}
+
+	// Now walk all recursive pins to check for indirect pins
+	var checkChildren func(key.Key, key.Key) error
+	checkChildren = func(rk key.Key, parentKey key.Key) error {
+		parent, err := p.dserv.Get(context.Background(), parentKey)
+		if err != nil {
+			return err
+		}
+		for _, lnk := range parent.Links {
+			k := key.Key(lnk.Hash)
+
+			if _, found := toCheck[k]; found {
+				pinned = append(pinned,
+					Pinned{Key: k, Mode: Indirect, Via: rk})
+				delete(toCheck, k)
+			}
+
+			err := checkChildren(rk, k)
+			if err != nil {
+				return err
+			}
+
+			if len(toCheck) == 0 {
+				return nil
+			}
+		}
+		return nil
+	}
+	for _, rk := range p.recursePin.GetKeys() {
+		err := checkChildren(rk, rk)
+		if err != nil {
+			return nil, err
+		}
+		if len(toCheck) == 0 {
+			break
+		}
+	}
+
+	// Anything left in toCheck is not pinned
+	for k, _ := range toCheck {
+		pinned = append(pinned, Pinned{Key: k, Mode: NotPinned})
+	}
+
+	return pinned, nil
+}
+
 func (p *pinner) RemovePinWithMode(key key.Key, mode PinMode) {
 	p.lock.Lock()
 	defer p.lock.Unlock()

test/sharness/t0050-block.sh

@@ -10,17 +10,26 @@ test_description="Test block command"
 
 test_init_ipfs
 
+HASH="QmRKqGMAM6EZngbpjSqrvYzq5Qd8b1bSWymjSUY9zQSNDk"
+
+#
+# "block put tests"
+#
+
 test_expect_success "'ipfs block put' succeeds" '
 	echo "Hello Mars!" >expected_in &&
 	ipfs block put <expected_in >actual_out
 '
 
 test_expect_success "'ipfs block put' output looks good" '
-	HASH="QmRKqGMAM6EZngbpjSqrvYzq5Qd8b1bSWymjSUY9zQSNDk" &&
 	echo "$HASH" >expected_out &&
 	test_cmp expected_out actual_out
 '
 
+#
+# "block get" tests
+#
+
 test_expect_success "'ipfs block get' succeeds" '
 	ipfs block get $HASH >actual_in
 '
@@ -29,16 +38,141 @@ test_expect_success "'ipfs block get' output looks good" '
 	test_cmp expected_in actual_in
 '
 
+#
+# "block stat" tests
+#
+
 test_expect_success "'ipfs block stat' succeeds" '
   ipfs block stat $HASH >actual_stat
 '
 
-test_expect_success "'ipfs block get' output looks good" '
+test_expect_success "'ipfs block stat' output looks good" '
   echo "Key: $HASH" >expected_stat &&
   echo "Size: 12" >>expected_stat &&
   test_cmp expected_stat actual_stat
 '
 
+#
+# "block rm" tests
+#
+
+test_expect_success "'ipfs block rm' succeeds" '
+  ipfs block rm $HASH >actual_rm
+'
+
+test_expect_success "'ipfs block rm' output looks good" '
+  echo "removed $HASH" > expected_rm &&
+  test_cmp expected_rm actual_rm
+'
+
+test_expect_success "'ipfs block rm' block actually removed" '
+  test_must_fail ipfs block stat $HASH
+'
+
+DIRHASH=QmdWmVmM6W2abTgkEfpbtA1CJyTWS2rhuUB9uP1xV8Uwtf
+FILE1HASH=Qmae3RedM7SNkWGsdzYzsr6svmsFdsva4WoTvYYsWhUSVz
+FILE2HASH=QmUtkGLvPf63NwVzLPKPUYgwhn8ZYPWF6vKWN3fZ2amfJF
+FILE3HASH=Qmesmmf1EEG1orJb6XdK6DabxexsseJnCfw8pqWgonbkoj
+
+test_expect_success "add and pin directory" '
+  mkdir adir &&
+  echo "file1" > adir/file1 &&
+  echo "file2" > adir/file2 &&
+  echo "file3" > adir/file3 &&
+  ipfs add -r adir
+  ipfs pin add -r $DIRHASH
+'
+
+test_expect_success "can't remove pinned block" '
+  test_must_fail ipfs block rm $DIRHASH 2> block_rm_err
+'
+
+test_expect_success "can't remove pinned block: output looks good" '
+  grep -q "$DIRHASH: pinned: recursive" block_rm_err
+'
+
+test_expect_success "can't remove indirectly pinned block" '
+  test_must_fail ipfs block rm $FILE1HASH 2> block_rm_err
+'
+
+test_expect_success "can't remove indirectly pinned block: output looks good" '
+  grep -q "$FILE1HASH: pinned via $DIRHASH" block_rm_err
+'
+
+test_expect_success "remove pin" '
+  ipfs pin rm -r $DIRHASH
+'
+
+test_expect_success "multi-block 'ipfs block rm' succeeds" '
+  ipfs block rm $FILE1HASH $FILE2HASH $FILE3HASH > actual_rm
+'
+
+test_expect_success "multi-block 'ipfs block rm' output looks good" '
+  grep -F -q "removed $FILE1HASH" actual_rm &&
+  grep -F -q "removed $FILE2HASH" actual_rm &&
+  grep -F -q "removed $FILE3HASH" actual_rm
+'
+
+test_expect_success "'add some blocks' succeeds" '
+        echo "Hello Mars!" | ipfs block put &&
+        echo "Hello Venus!" | ipfs block put
+'
+
+test_expect_success "add and pin directory" '
+  ipfs add -r adir
+  ipfs pin add -r $DIRHASH
+'
+
+HASH=QmRKqGMAM6EZngbpjSqrvYzq5Qd8b1bSWymjSUY9zQSNDk
+HASH2=QmdnpnsaEj69isdw5sNzp3h3HkaDz7xKq7BmvFFBzNr5e7
+RANDOMHASH=QRmKqGMAM6EbngbZjSqrvYzq5Qd8b1bSWymjSUY9zQSNDq
+
+test_expect_success "multi-block 'ipfs block rm' mixed" '
+  test_must_fail ipfs block rm $FILE1HASH $DIRHASH $HASH $FILE3HASH $RANDOMHASH $HASH2 2> block_rm_err
+'
+
+test_expect_success "pinned block not removed" '
+  ipfs block stat $FILE1HASH &&
+  ipfs block stat $FILE3HASH
+'
+
+test_expect_success "non-pinned blocks removed" '
+  test_must_fail ipfs block stat $HASH &&
+  test_must_fail ipfs block stat $HASH2
+'
+
+test_expect_success "error reported on removing non-existent block" '
+  grep -q "cannot remove $RANDOMHASH" block_rm_err
+'
+
+test_expect_success "'add some blocks' succeeds" '
+        echo "Hello Mars!" | ipfs block put &&
+        echo "Hello Venus!" | ipfs block put
+'
+
+test_expect_success "multi-block 'ipfs block rm -f' with non existent blocks succeed" '
+  ipfs block rm -f $HASH $RANDOMHASH $HASH2
+'
+
+test_expect_success "existent blocks removed" '
+  test_must_fail ipfs block stat $HASH &&
+  test_must_fail ipfs block stat $HASH2
+'
+
+test_expect_success "'add some blocks' succeeds" '
+        echo "Hello Mars!" | ipfs block put &&
+        echo "Hello Venus!" | ipfs block put
+'
+
+test_expect_success "multi-block 'ipfs block rm -q' produces no output" '
+  ipfs block rm -q $HASH $HASH2 > block_rm_out &&
+  test ! -s block_rm_out
+'
+
+#
+# Misc tests
+#
+
 test_expect_success "'ipfs block stat' with nothing from stdin doesnt crash" '
 	test_expect_code 1 ipfs block stat < /dev/null 2> stat_out
 '