move GC code into core/repo, and add sharness test

core/commands/ping.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"fmt"
 	"io"
+	"reflect"
 	"strings"
 	"time"
 
@@ -46,6 +47,7 @@ trip latency information.
 		cmds.Text: func(res cmds.Response) (io.Reader, error) {
 			outChan, ok := res.Output().(<-chan interface{})
 			if !ok {
+				fmt.Println(reflect.TypeOf(res.Output()))
 				return nil, u.ErrCast()
 			}
 
@@ -103,63 +105,64 @@ trip latency information.
 			numPings = val
 		}
 
-		outChan := make(chan interface{})
-
-		go pingPeer(ctx, n, peerID, numPings, outChan)
-
+		outChan := pingPeer(ctx, n, peerID, numPings)
 		return outChan, nil
 	},
 	Type: PingResult{},
 }
 
-func pingPeer(ctx context.Context, n *core.IpfsNode, pid peer.ID, numPings int, outChan chan interface{}) {
-	defer close(outChan)
+func pingPeer(ctx context.Context, n *core.IpfsNode, pid peer.ID, numPings int) <-chan interface{} {
+	outChan := make(chan interface{})
+	go func() {
+		defer close(outChan)
 
-	if len(n.Peerstore.Addresses(pid)) == 0 {
-		// Make sure we can find the node in question
-		outChan <- &PingResult{
-			Text: fmt.Sprintf("Looking up peer %s", pid.Pretty()),
-		}
+		if len(n.Peerstore.Addresses(pid)) == 0 {
+			// Make sure we can find the node in question
+			outChan <- &PingResult{
+				Text: fmt.Sprintf("Looking up peer %s", pid.Pretty()),
+			}
 
-		ctx, _ := context.WithTimeout(ctx, kPingTimeout)
-		p, err := n.Routing.FindPeer(ctx, pid)
-		if err != nil {
-			outChan <- &PingResult{Text: fmt.Sprintf("Peer lookup error: %s", err)}
-			return
+			ctx, _ := context.WithTimeout(ctx, kPingTimeout)
+			p, err := n.Routing.FindPeer(ctx, pid)
+			if err != nil {
+				outChan <- &PingResult{Text: fmt.Sprintf("Peer lookup error: %s", err)}
+				return
+			}
+			n.Peerstore.AddPeerInfo(p)
 		}
-		n.Peerstore.AddPeerInfo(p)
-	}
 
-	outChan <- &PingResult{Text: fmt.Sprintf("PING %s.", pid.Pretty())}
+		outChan <- &PingResult{Text: fmt.Sprintf("PING %s.", pid.Pretty())}
 
-	var done bool
-	var total time.Duration
-	for i := 0; i < numPings && !done; i++ {
-		select {
-		case <-ctx.Done():
-			done = true
-			continue
-		default:
-		}
+		var done bool
+		var total time.Duration
+		for i := 0; i < numPings && !done; i++ {
+			select {
+			case <-ctx.Done():
+				done = true
+				continue
+			default:
+			}
 
-		ctx, _ := context.WithTimeout(ctx, kPingTimeout)
-		took, err := n.Routing.Ping(ctx, pid)
-		if err != nil {
-			log.Errorf("Ping error: %s", err)
-			outChan <- &PingResult{Text: fmt.Sprintf("Ping error: %s", err)}
-			break
+			ctx, _ := context.WithTimeout(ctx, kPingTimeout)
+			took, err := n.Routing.Ping(ctx, pid)
+			if err != nil {
+				log.Errorf("Ping error: %s", err)
+				outChan <- &PingResult{Text: fmt.Sprintf("Ping error: %s", err)}
+				break
+			}
+			outChan <- &PingResult{
+				Success: true,
+				Time:    took,
+			}
+			total += took
+			time.Sleep(time.Second)
 		}
+		averagems := total.Seconds() * 1000 / float64(numPings)
 		outChan <- &PingResult{
-			Success: true,
-			Time:    took,
+			Text: fmt.Sprintf("Average latency: %.2fms", averagems),
 		}
-		total += took
-		time.Sleep(time.Second)
-	}
-	averagems := total.Seconds() * 1000 / float64(numPings)
-	outChan <- &PingResult{
-		Text: fmt.Sprintf("Average latency: %.2fms", averagems),
-	}
+	}()
+	return outChan
 }
 
 func ParsePeerParam(text string) (ma.Multiaddr, peer.ID, error) {

core/commands/repo.go

@@ -3,10 +3,11 @@ package commands
 import (
 	"bytes"
 	"fmt"
+	"io"
+
 	cmds "github.com/jbenet/go-ipfs/commands"
-	"github.com/jbenet/go-ipfs/core"
+	corerepo "github.com/jbenet/go-ipfs/core/repo"
 	u "github.com/jbenet/go-ipfs/util"
-	"io"
 )
 
 var RepoCmd = &cmds.Command{
@@ -22,10 +23,6 @@ var RepoCmd = &cmds.Command{
 	},
 }
 
-type KeyRemoved struct {
-	Key u.Key
-}
-
 var repoGcCmd = &cmds.Command{
 	Helptext: cmds.HelpText{
 		Tagline: "Perform a garbage collection sweep on the repo",
@@ -45,19 +42,17 @@ order to reclaim hard disk space.
 			return nil, err
 		}
 
-		keychan, err := n.Blockstore.AllKeysChan(req.Context().Context, 0, 1<<16)
+		outChan, err := corerepo.GarbageCollectBlockstore(n, req.Context().Context)
 		if err != nil {
 			return nil, err
 		}
 
-		outChan := make(chan interface{})
-		go GarbageCollectBlockstore(n, keychan, outChan)
 		return outChan, nil
 	},
-	Type: KeyRemoved{},
+	Type: corerepo.KeyRemoved{},
 	Marshalers: cmds.MarshalerMap{
 		cmds.Text: func(res cmds.Response) (io.Reader, error) {
-			outChan, ok := res.Output().(chan interface{})
+			outChan, ok := res.Output().(<-chan interface{})
 			if !ok {
 				return nil, u.ErrCast()
 			}
@@ -68,7 +63,7 @@ order to reclaim hard disk space.
 			}
 
 			marshal := func(v interface{}) (io.Reader, error) {
-				obj, ok := v.(*KeyRemoved)
+				obj, ok := v.(*corerepo.KeyRemoved)
 				if !ok {
 					return nil, u.ErrCast()
 				}
@@ -89,16 +84,3 @@ order to reclaim hard disk space.
 		},
 	},
 }
-
-func GarbageCollectBlockstore(n *core.IpfsNode, keychan <-chan u.Key, output chan interface{}) {
-	defer close(output)
-	for k := range keychan {
-		if !n.Pinning.IsPinned(k) {
-			err := n.Blockstore.DeleteBlock(k)
-			if err != nil {
-				log.Errorf("Error removing key from blockstore: %s", err)
-			}
-			output <- &KeyRemoved{k}
-		}
-	}
-}

core/repo/gc.go

+package corerepo
+
+import (
+	context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"
+	"github.com/jbenet/go-ipfs/core"
+	u "github.com/jbenet/go-ipfs/util"
+
+	eventlog "github.com/jbenet/go-ipfs/thirdparty/eventlog"
+)
+
+var log = eventlog.Logger("corerepo")
+
+type KeyRemoved struct {
+	Key u.Key
+}
+
+func GarbageCollectBlockstore(n *core.IpfsNode, ctx context.Context) (<-chan interface{}, error) {
+
+	keychan, err := n.Blockstore.AllKeysChan(ctx, 0, 1<<16)
+	if err != nil {
+		return nil, err
+	}
+
+	output := make(chan interface{})
+	go func() {
+		defer close(output)
+		for {
+			select {
+			case k, ok := <-keychan:
+				if !ok {
+					return
+				}
+				if !n.Pinning.IsPinned(k) {
+					err := n.Blockstore.DeleteBlock(k)
+					if err != nil {
+						log.Errorf("Error removing key from blockstore: %s", err)
+					}
+					select {
+					case output <- &KeyRemoved{k}:
+					case <-ctx.Done():
+					}
+				}
+			case <-ctx.Done():
+				return
+			}
+		}
+	}()
+	return output, nil
+}

test/sharness/t0080-repo.sh

@@ -21,7 +21,11 @@ test_expect_success "added file was pinned" '
 	ipfs pin ls -type=recursive | grep `cat hashfile`
 '
 
-# TODO: run gc, then ipfs cat file, should still be there
+test_expect_success "'ipfs repo gc' doesnt remove file" '
+	ipfs repo gc
+	ipfs cat `cat hashfile` > out
+	test_cmp out afile
+'
 
 test_expect_success "'ipfs pin rm' succeeds" '
 	echo unpinned `cat hashfile` > expected1
@@ -64,6 +68,18 @@ test_expect_success "remove direct pin" '
 	test_cmp expected6 actual6
 '
 
+test_expect_success "'ipfs repo gc' removes file" '
+	echo removed `cat hashfile` > expected7
+	ipfs repo gc > actual7
+	test_cmp expected7 actual7
+'
+
+test_expect_success "'ipfs refs local' no longer shows file" '
+	echo -n "" > expected8
+	ipfs refs local > actual8
+	test_cmp expected8 actual8
+'
+
 
 test_kill_ipfs_daemon