Merge pull request #5010 from ipfs/feat/diskusage2

Efficient "repo stat" (DiskUsage) and "--size-only" flag

core/commands/repo.go

@@ -150,14 +150,20 @@ var repoStatCmd = &cmds.Command{
 	Helptext: cmdkit.HelpText{
 		Tagline: "Get stats for the currently used repo.",
 		ShortDescription: `
-'ipfs repo stat' is a plumbing command that will scan the local
-set of stored objects and print repo statistics. It outputs to stdout:
+'ipfs repo stat' provides information about the local set of
+stored objects. It outputs:
+
+RepoSize        int Size in bytes that the repo is currently taking.
+StorageMax      string Maximum datastore size (from configuration)
 NumObjects      int Number of objects in the local repo.
 RepoPath        string The path to the repo being currently used.
-RepoSize        int Size in bytes that the repo is currently taking.
 Version         string The repo version.
 `,
 	},
+	Options: []cmdkit.Option{
+		cmdkit.BoolOption("size-only", "Only report RepoSize and StorageMax."),
+		cmdkit.BoolOption("human", "Output sizes in MiB."),
+	},
 	Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) {
 		n, err := GetNode(env)
 		if err != nil {
@@ -165,18 +171,28 @@ Version         string The repo version.
 			return
 		}
 
-		stat, err := corerepo.RepoStat(n, req.Context)
+		sizeOnly, _ := req.Options["size-only"].(bool)
+		if sizeOnly {
+			sizeStat, err := corerepo.RepoSize(req.Context, n)
+			if err != nil {
+				res.SetError(err, cmdkit.ErrNormal)
+				return
+			}
+			cmds.EmitOnce(res, &corerepo.Stat{
+				SizeStat: sizeStat,
+			})
+			return
+		}
+
+		stat, err := corerepo.RepoStat(req.Context, n)
 		if err != nil {
 			res.SetError(err, cmdkit.ErrNormal)
 			return
 		}
 
-		cmds.EmitOnce(res, stat)
-	},
-	Options: []cmdkit.Option{
-		cmdkit.BoolOption("human", "Output RepoSize in MiB."),
+		cmds.EmitOnce(res, &stat)
 	},
-	Type: corerepo.Stat{},
+	Type: &corerepo.Stat{},
 	Encoders: cmds.EncoderMap{
 		cmds.Text: cmds.MakeEncoder(func(req *cmds.Request, w io.Writer, v interface{}) error {
 			stat, ok := v.(*corerepo.Stat)
@@ -184,31 +200,34 @@ Version         string The repo version.
 				return e.TypeErr(stat, v)
 			}
 
-			human, _ := req.Options["human"].(bool)
-
 			wtr := tabwriter.NewWriter(w, 0, 0, 1, ' ', 0)
+			defer wtr.Flush()
 
-			fmt.Fprintf(wtr, "NumObjects:\t%d\n", stat.NumObjects)
-			sizeInMiB := stat.RepoSize / (1024 * 1024)
-			if human && sizeInMiB > 0 {
-				fmt.Fprintf(wtr, "RepoSize (MiB):\t%d\n", sizeInMiB)
-			} else {
-				fmt.Fprintf(wtr, "RepoSize:\t%d\n", stat.RepoSize)
-			}
-			if stat.StorageMax != corerepo.NoLimit {
-				maxSizeInMiB := stat.StorageMax / (1024 * 1024)
-				if human && maxSizeInMiB > 0 {
-					fmt.Fprintf(wtr, "StorageMax (MiB):\t%d\n", maxSizeInMiB)
+			human, _ := req.Options["human"].(bool)
+			sizeOnly, _ := req.Options["size-only"].(bool)
+
+			printSize := func(name string, size uint64) {
+				sizeInMiB := size / (1024 * 1024)
+				if human && sizeInMiB > 0 {
+					fmt.Fprintf(wtr, "%s (MiB):\t%d\n", name, sizeInMiB)
 				} else {
-					fmt.Fprintf(wtr, "StorageMax:\t%d\n", stat.StorageMax)
+					fmt.Fprintf(wtr, "%s:\t%d\n", name, size)
 				}
 			}
-			fmt.Fprintf(wtr, "RepoPath:\t%s\n", stat.RepoPath)
-			fmt.Fprintf(wtr, "Version:\t%s\n", stat.Version)
-			wtr.Flush()
 
-			return nil
+			if !sizeOnly {
+				fmt.Fprintf(wtr, "NumObjects:\t%d\n", stat.NumObjects)
+			}
+
+			printSize("RepoSize", stat.RepoSize)
+			printSize("StorageMax", stat.StorageMax)
 
+			if !sizeOnly {
+				fmt.Fprintf(wtr, "RepoPath:\t%s\n", stat.RepoPath)
+				fmt.Fprintf(wtr, "Version:\t%s\n", stat.Version)
+			}
+
+			return nil
 		}),
 	},
 }

core/corerepo/stat.go

@@ -5,34 +5,40 @@ import (
 	"math"
 
 	context "context"
+
 	"github.com/ipfs/go-ipfs/core"
 	fsrepo "github.com/ipfs/go-ipfs/repo/fsrepo"
 
 	humanize "gx/ipfs/QmPSBJL4momYnE7DcUyk2DVhD6rH488ZmHBGLbxNdhU44K/go-humanize"
 )
 
+// SizeStat wraps information about the repository size and its limit.
+type SizeStat struct {
+	RepoSize   uint64 // size in bytes
+	StorageMax uint64 // size in bytes
+}
+
+// Stat wraps information about the objects stored on disk.
 type Stat struct {
+	SizeStat
 	NumObjects uint64
-	RepoSize   uint64 // size in bytes
 	RepoPath   string
 	Version    string
-	StorageMax uint64 // size in bytes
 }
 
 // NoLimit represents the value for unlimited storage
 const NoLimit uint64 = math.MaxUint64
 
-func RepoStat(n *core.IpfsNode, ctx context.Context) (*Stat, error) {
-	r := n.Repo
-
-	usage, err := r.GetStorageUsage()
+// RepoStat returns a *Stat object with all the fields set.
+func RepoStat(ctx context.Context, n *core.IpfsNode) (Stat, error) {
+	sizeStat, err := RepoSize(ctx, n)
 	if err != nil {
-		return nil, err
+		return Stat{}, err
 	}
 
 	allKeys, err := n.Blockstore.AllKeysChan(ctx)
 	if err != nil {
-		return nil, err
+		return Stat{}, err
 	}
 
 	count := uint64(0)
@@ -42,27 +48,44 @@ func RepoStat(n *core.IpfsNode, ctx context.Context) (*Stat, error) {
 
 	path, err := fsrepo.BestKnownPath()
 	if err != nil {
-		return nil, err
+		return Stat{}, err
 	}
 
+	return Stat{
+		SizeStat: SizeStat{
+			RepoSize:   sizeStat.RepoSize,
+			StorageMax: sizeStat.StorageMax,
+		},
+		NumObjects: count,
+		RepoPath:   path,
+		Version:    fmt.Sprintf("fs-repo@%d", fsrepo.RepoVersion),
+	}, nil
+}
+
+// RepoSize returns a *Stat object with the RepoSize and StorageMax fields set.
+func RepoSize(ctx context.Context, n *core.IpfsNode) (SizeStat, error) {
+	r := n.Repo
+
 	cfg, err := r.Config()
 	if err != nil {
-		return nil, err
+		return SizeStat{}, err
+	}
+
+	usage, err := r.GetStorageUsage()
+	if err != nil {
+		return SizeStat{}, err
 	}
 
 	storageMax := NoLimit
 	if cfg.Datastore.StorageMax != "" {
 		storageMax, err = humanize.ParseBytes(cfg.Datastore.StorageMax)
 		if err != nil {
-			return nil, err
+			return SizeStat{}, err
 		}
 	}
 
-	return &Stat{
-		NumObjects: count,
+	return SizeStat{
 		RepoSize:   usage,
-		RepoPath:   path,
-		Version:    fmt.Sprintf("fs-repo@%d", fsrepo.RepoVersion),
 		StorageMax: storageMax,
 	}, nil
 }

repo/fsrepo/fsrepo.go

@@ -27,6 +27,7 @@ import (
 	ma "gx/ipfs/QmYmsdtJ3HsodkePE3eU3TsCaP2YvPZJ4LoXnNkDE5Tpt7/go-multiaddr"
 	lockfile "gx/ipfs/QmYzCZUe9CBDkyPNPcRNqXQK8KKhtUfXvc88PkFujAEJPe/go-fs-lock"
 	logging "gx/ipfs/QmcVVHfdyv15GVPk7NrxdWjh2hLVccXnoD8j2tyQShiXJb/go-log"
+	ds "gx/ipfs/QmeiCcJfDW1GJnWUArudsv5rQsihpi4oyddPhdqo3CfX6i/go-datastore"
 )
 
 // LockFile is the filename of the repo lock, relative to config dir
@@ -674,29 +675,7 @@ func (r *FSRepo) Datastore() repo.Datastore {
 
 // GetStorageUsage computes the storage space taken by the repo in bytes
 func (r *FSRepo) GetStorageUsage() (uint64, error) {
-	pth, err := config.PathRoot()
-	if err != nil {
-		return 0, err
-	}
-
-	pth, err = filepath.EvalSymlinks(pth)
-	if err != nil {
-		log.Debugf("filepath.EvalSymlinks error: %s", err)
-		return 0, err
-	}
-
-	var du uint64
-	err = filepath.Walk(pth, func(p string, f os.FileInfo, err error) error {
-		if err != nil {
-			log.Debugf("filepath.Walk error: %s", err)
-			return nil
-		}
-		if f != nil {
-			du += uint64(f.Size())
-		}
-		return nil
-	})
-	return du, err
+	return ds.DiskUsage(r.Datastore())
 }
 
 func (r *FSRepo) SwarmKey() ([]byte, error) {

test/sharness/t0080-repo.sh

@@ -245,6 +245,18 @@ test_expect_success "repo stats are updated correctly" '
   test $(get_field_num "RepoSize" repo-stats-2) -ge $(get_field_num "RepoSize" repo-stats)
 '
 
+test_expect_success "'ipfs repo stat --size-only' succeeds" '
+  ipfs repo stat --size-only > repo-stats-size-only
+'
+
+test_expect_success "repo stats came out correct for --size-only" '
+  grep "RepoSize" repo-stats-size-only &&
+  grep "StorageMax" repo-stats-size-only &&
+  grep -v "RepoPath" repo-stats-size-only &&
+  grep -v "NumObjects" repo-stats-size-only &&
+  grep -v "Version" repo-stats-size-only
+'
+
 test_expect_success "'ipfs repo version' succeeds" '
   ipfs repo version > repo-version
 '