commands: repo fsck (#2597)

* Adds repo fsck subcommand

Fixes #2457

License: MIT
Signed-off-by: Mike Pfister <pfista@gmail.com>

* Checks for error on file deletion

License: MIT
Signed-off-by: Mike Pfister <pfista@gmail.com>

* Checks if node is online

License: MIT
Signed-off-by: Mike Pfister <pfista@gmail.com>

* Update error checking

License: MIT
Signed-off-by: Michael Pfister <pfista@gmail.com>

* Prevents command from running while daemon is running

License: MIT
Signed-off-by: Michael Pfister <pfista@gmail.com>

* Add newline to command output message

License: MIT
Signed-off-by: Michael Pfister <pfista@gmail.com>

* removing superfluous error

License: MIT
Signed-off-by: Michael Pfister <pfista@gmail.com>

* Adds sharness test for repo fsck command

License: MIT
Signed-off-by: Michael Pfister <pfista@gmail.com>

* Ignore warning if file doesn't exist

License: MIT
Signed-off-by: Michael Pfister <pfista@gmail.com>

* Updating message output

License: MIT
Signed-off-by: Michael Pfister <pfista@gmail.com>

* adding debug statements

License: MIT
Signed-off-by: Michael Pfister <pfista@gmail.com>

* update and add fsck sharness tests

License: MIT
Signed-off-by: Michael Pfister <pfista@gmail.com>

* updating comments

License: MIT
Signed-off-by: Michael Pfister <pfista@gmail.com>

* Use printf in test

Using printf prevents a newline from being printed to the api test file. When
the newline was present, multiaddr threw errors  trying to parse the api address
to an integer since the newline character was present.

License: MIT
Signed-off-by: Michael Pfister <pfista@gmail.com>

* updating tests

License: MIT
Signed-off-by: Michael Pfister <pfista@gmail.com>

* removing commented code

License: MIT
Signed-off-by: Michael Pfister <pfista@gmail.com>

cmd/ipfs/init.go

@@ -51,6 +51,7 @@ at ~/.ipfs. To change the repo location, set the $IPFS_PATH environment variable
 
 		log.Info("checking if daemon is running...")
 		if daemonLocked {
+			log.Debug("Ipfs daemon is running.")
 			e := "ipfs daemon is running. please stop it to run this command"
 			return cmds.ClientError(e)
 		}

cmd/ipfs/ipfs.go

@@ -105,4 +105,5 @@ var cmdDetailsMap = map[*cmds.Command]cmdDetails{
 	commands.CommandsDaemonCmd: {doesNotUseRepo: true},
 	commands.VersionCmd:        {doesNotUseConfigAsInput: true, doesNotUseRepo: true}, // must be permitted to run before init
 	commands.LogCmd:            {cannotRunOnClient: true},
+	commands.RepoFsckCmd:       {cannotRunOnDaemon: true},
 }

cmd/ipfs/main.go

@@ -417,9 +417,10 @@ func commandShouldRunOnDaemon(details cmdDetails, req cmds.Request, root *cmds.C
 		return nil, err
 	}
 
-	if client != nil { // api file exists
+	if client != nil {
 		if details.cannotRunOnDaemon {
 			// check if daemon locked. legacy error text, for now.
+			log.Debugf("Command cannot run on daemon. Checking if daemon is locked")
 			if daemonLocked, _ := fsrepo.LockedByOtherProcess(req.InvocContext().ConfigRoot); daemonLocked {
 				return nil, cmds.ClientError("ipfs daemon is running. please stop it to run this command")
 			}

core/commands/repo.go

@@ -5,8 +5,12 @@ import (
 	"fmt"
 	cmds "github.com/ipfs/go-ipfs/commands"
 	corerepo "github.com/ipfs/go-ipfs/core/corerepo"
+	config "github.com/ipfs/go-ipfs/repo/config"
+	lockfile "github.com/ipfs/go-ipfs/repo/fsrepo/lock"
 	u "gx/ipfs/QmZNVWh8LLjAavuQ2JXuFmuYH3C11xo988vSgp7UQrTRj1/go-ipfs-util"
 	"io"
+	"os"
+	"path/filepath"
 )
 
 var RepoCmd = &cmds.Command{
@@ -20,6 +24,7 @@ var RepoCmd = &cmds.Command{
 	Subcommands: map[string]*cmds.Command{
 		"gc":   repoGcCmd,
 		"stat": repoStatCmd,
+		"fsck": RepoFsckCmd,
 	},
 }
 
@@ -32,7 +37,6 @@ set of stored objects and remove ones that are not pinned in
 order to reclaim hard disk space.
 `,
 	},
-
 	Options: []cmds.Option{
 		cmds.BoolOption("quiet", "q", "Write minimal output."),
 	},
@@ -152,3 +156,55 @@ RepoSize        int Size in bytes that the repo is currently taking.
 		},
 	},
 }
+
+var RepoFsckCmd = &cmds.Command{
+	Helptext: cmds.HelpText{
+		Tagline: "Removes repo lockfiles",
+		ShortDescription: `
+'ipfs repo fsck' is a plumbing command that will remove repo and level db
+lockfiles, as well as the api file. This command can only run when no ipfs
+daemons are running.
+`,
+	},
+	Run: func(req cmds.Request, res cmds.Response) {
+		configRoot := req.InvocContext().ConfigRoot
+
+		dsPath, err := config.DataStorePath(configRoot)
+		if err != nil {
+			res.SetError(err, cmds.ErrNormal)
+			return
+		}
+
+		dsLockFile := filepath.Join(dsPath, "LOCK") // TODO: get this lockfile programmatically
+		repoLockFile := filepath.Join(configRoot, lockfile.LockFile)
+		apiFile := filepath.Join(configRoot, "api") // TODO: get this programmatically
+
+		log.Infof("Removing repo lockfile: %s", repoLockFile)
+		log.Infof("Removing datastore lockfile: %s", dsLockFile)
+		log.Infof("Removing api file: %s", apiFile)
+
+		err = os.Remove(repoLockFile)
+		if err != nil && !os.IsNotExist(err) {
+			res.SetError(err, cmds.ErrNormal)
+			return
+		}
+		err = os.Remove(dsLockFile)
+		if err != nil && !os.IsNotExist(err) {
+			res.SetError(err, cmds.ErrNormal)
+			return
+		}
+		err = os.Remove(apiFile)
+		if err != nil && !os.IsNotExist(err) {
+			res.SetError(err, cmds.ErrNormal)
+			return
+		}
+
+		s := "Lockfiles have been removed."
+		log.Info(s)
+		res.SetOutput(&MessageOutput{s + "\n"})
+	},
+	Type: MessageOutput{},
+	Marshalers: cmds.MarshalerMap{
+		cmds.Text: MessageTextMarshaler,
+	},
+}

repo/fsrepo/fsrepo.go

@@ -259,8 +259,11 @@ func Remove(repoPath string) error {
 // process. If true, then the repo cannot be opened by this process.
 func LockedByOtherProcess(repoPath string) (bool, error) {
 	repoPath = filepath.Clean(repoPath)
-	// NB: the lock is only held when repos are Open
-	return lockfile.Locked(repoPath)
+	locked, err := lockfile.Locked(repoPath)
+	if locked {
+		log.Debugf("(%t)<->Lock is held at %s", locked, repoPath)
+	}
+	return locked, err
 }
 
 // APIAddr returns the registered API addr, according to the api file
@@ -360,7 +363,7 @@ func (r *FSRepo) Close() error {
 	}
 
 	err := os.Remove(filepath.Join(r.path, apiFile))
-	if err != nil {
+	if err != nil && !os.IsNotExist(err) {
 		log.Warning("error removing api file: ", err)
 	}
 

repo/fsrepo/lock/lock.go

@@ -10,12 +10,16 @@ import (
 
 	lock "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/camlistore/lock"
 	"gx/ipfs/QmZNVWh8LLjAavuQ2JXuFmuYH3C11xo988vSgp7UQrTRj1/go-ipfs-util"
+	logging "gx/ipfs/Qmazh5oNUVsDZTs2g59rq8aYQqwpss8tcUWQzor5sCCEuH/go-log"
 )
 
 // LockFile is the filename of the repo lock, relative to config dir
 // TODO rename repo lock and hide name
 const LockFile = "repo.lock"
 
+// log is the fsrepo logger
+var log = logging.Logger("lock")
+
 func errPerm(path string) error {
 	return fmt.Errorf("failed to take lock at %s: permission denied", path)
 }
@@ -26,29 +30,36 @@ func Lock(confdir string) (io.Closer, error) {
 }
 
 func Locked(confdir string) (bool, error) {
+	log.Debugf("Checking lock")
 	if !util.FileExists(path.Join(confdir, LockFile)) {
+		log.Debugf("File doesn't exist: %s", path.Join(confdir, LockFile))
 		return false, nil
 	}
 	if lk, err := Lock(confdir); err != nil {
 		// EAGAIN == someone else has the lock
 		if err == syscall.EAGAIN {
+			log.Debugf("Someone else has the lock: %s", path.Join(confdir, LockFile))
 			return true, nil
 		}
 		if strings.Contains(err.Error(), "can't Lock file") {
+			log.Debugf("Can't lock file: %s.\n reason: %s", path.Join(confdir, LockFile), err.Error())
 			return true, nil
 		}
 
 		// lock fails on permissions error
 		if os.IsPermission(err) {
+			log.Debugf("Lock fails on permissions error")
 			return false, errPerm(confdir)
 		}
 		if isLockCreatePermFail(err) {
+			log.Debugf("Lock fails on permissions error")
 			return false, errPerm(confdir)
 		}
 
 		// otherwise, we cant guarantee anything, error out
 		return false, err
 	} else {
+		log.Debugf("No one has a lock")
 		lk.Close()
 		return false, nil
 	}

test/sharness/t0083-repo-fsck.sh

+#!/bin/sh
+#
+# Copyright (c) 2016 Mike Pfister 
+# MIT Licensed; see the LICENSE file in this repository.
+#
+
+test_description="Test ipfs repo fsck operations"
+
+. lib/test-lib.sh
+
+test_init_ipfs
+
+#############################
+# Test without daemon running
+############################# 
+# Note: if api file isn't present we can assume the daemon isn't running
+
+# Try with all lock files present: repo.lock, api, and datastore/LOCK with
+# repo.lock and datastore/LOCK being empty
+test_expect_success "'ipfs repo fsck' succeeds with no daemon running empty
+repo.lock" '
+    mkdir -p $IPFS_PATH &&
+    mkdir -p $IPFS_PATH/datastore &&
+    touch $IPFS_PATH/datastore/LOCK &&
+    touch $IPFS_PATH/repo.lock &&
+    printf "/ip4/127.0.0.1/tcp/5001" > $IPFS_PATH/api &&
+    ipfs repo fsck > fsck_out_actual1
+'
+test_expect_success "'ipfs repo fsck' output looks good with no daemon" '
+  grep "Lockfiles have been removed." fsck_out_actual1
+'
+
+# Make sure the files are actually removed
+test_expect_success "'ipfs repo fsck' confirm file deletion" '
+  test ! -e "$IPFS_PATH/repo.lock" &&
+  test ! -e "$IPFS_PATH/datastore/LOCK" &&
+  test ! -e "$IPFS_PATH/api"
+'
+
+# Try with all lock files present: repo.lock, api, and datastore/LOCK with
+# repo.lock is non-zero TODO: this test is broken until we find consensus on the
+# non-zero repo.lock issue
+test_expect_success "'ipfs repo fsck' succeeds with no daemon running non-zero
+repo.lock" '
+    mkdir -p $IPFS_PATH &&
+    printf ":D" > $IPFS_PATH/repo.lock &&
+    touch $IPFS_PATH/datastore/LOCK &&
+    ipfs repo fsck > fsck_out_actual1b
+'
+test_expect_success "'ipfs repo fsck' output looks good with no daemon" '
+  grep "Lockfiles have been removed." fsck_out_actual1b
+'
+
+# Make sure the files are actually removed
+test_expect_success "'ipfs repo fsck' confirm file deletion" '
+  test ! -e "$IPFS_PATH/repo.lock" &&
+  test ! -e "$IPFS_PATH/datastore/LOCK" &&
+  test ! -e "$IPFS_PATH/api"
+'
+
+########################
+# Test for partial locks
+######################## 
+
+# Try with locks api and datastore/LOCK
+test_expect_success "'ipfs repo fsck' succeeds partial lock" '
+    printf  "/ip4/127.0.0.1/tcp/5001" > $IPFS_PATH/api &&
+    touch $IPFS_PATH/datastore/LOCK &&
+    ipfs repo fsck > fsck_out_actual2
+'
+
+test_expect_success "'ipfs repo fsck' output looks good with no daemon" '
+  grep "Lockfiles have been removed." fsck_out_actual2
+'
+
+# Make sure the files are actually removed
+test_expect_success "'ipfs repo fsck' confirm file deletion" '
+  test ! -e "$IPFS_PATH/repo.lock" &&
+  test ! -e "$IPFS_PATH/datastore/LOCK" &&
+  test ! -e "$IPFS_PATH/api"
+'
+
+# Try with locks api and repo.lock
+test_expect_success "'ipfs repo fsck' succeeds partial lock" '
+    printf  "/ip4/127.0.0.1/tcp/5001" > $IPFS_PATH/api &&
+    touch $IPFS_PATH/repo.lock &&
+    ipfs repo fsck > fsck_out_actual3
+'
+
+test_expect_success "'ipfs repo fsck' output looks good with no daemon" '
+  grep "Lockfiles have been removed." fsck_out_actual3
+'
+
+# Make sure the files are actually removed
+test_expect_success "'ipfs repo fsck' confirm file deletion" '
+  test ! -e "$IPFS_PATH/repo.lock" &&
+  test ! -e "$IPFS_PATH/datastore/LOCK" &&
+  test ! -e "$IPFS_PATH/api"
+'
+
+# Try with locks repo.lock and datastore
+test_expect_success "'ipfs repo fsck' succeeds partial lock" '
+    touch $IPFS_PATH/repo.lock &&
+    touch $IPFS_PATH/datastore/LOCK &&
+    ipfs repo fsck > fsck_out_actual4
+'
+
+test_expect_success "'ipfs repo fsck' output looks good with no daemon" '
+  grep "Lockfiles have been removed." fsck_out_actual4
+'
+
+# Make sure the files are actually removed
+test_expect_success "'ipfs repo fsck' confirm file deletion" '
+  test ! -e "$IPFS_PATH/repo.lock" &&
+  test ! -e "$IPFS_PATH/datastore/LOCK" &&
+  test ! -e "$IPFS_PATH/api"
+'
+
+#######################
+# Test for single locks
+#######################
+
+# Try with single locks repo.lock
+test_expect_success "'ipfs repo fsck' succeeds partial lock" '
+    touch $IPFS_PATH/repo.lock &&
+    ipfs repo fsck > fsck_out_actual5
+'
+test_expect_success "'ipfs repo fsck' output looks good with no daemon" '
+  grep "Lockfiles have been removed." fsck_out_actual5
+'
+
+# Make sure the files are actually removed
+test_expect_success "'ipfs repo fsck' confirm file deletion" '
+  test ! -e "$IPFS_PATH/repo.lock" &&
+  test ! -e "$IPFS_PATH/datastore/LOCK" &&
+  test ! -e "$IPFS_PATH/api"
+'
+
+# Try with single locks datastore/LOCK
+test_expect_success "'ipfs repo fsck' succeeds partial lock" '
+    touch $IPFS_PATH/datastore/LOCK &&
+    ipfs repo fsck > fsck_out_actual6
+'
+test_expect_success "'ipfs repo fsck' output looks good with no daemon" '
+  grep "Lockfiles have been removed." fsck_out_actual6
+'
+
+# Make sure the files are actually removed
+test_expect_success "'ipfs repo fsck' confirm file deletion" '
+  test ! -e "$IPFS_PATH/repo.lock" &&
+  test ! -e "$IPFS_PATH/datastore/LOCK" &&
+  test ! -e "$IPFS_PATH/api"
+'
+
+# Try with single lock api
+test_expect_success "'ipfs repo fsck' succeeds partial lock" '
+  printf "/ip4/127.0.0.1/tcp/5001" > $IPFS_PATH/api &&
+  ipfs repo fsck > fsck_out_actual7
+'
+
+test_expect_success "'ipfs repo fsck' output looks good with no daemon" '
+  grep "Lockfiles have been removed." fsck_out_actual7
+'
+
+# Make sure the files are actually removed
+test_expect_success "'ipfs repo fsck' confirm file deletion" '
+  test ! -e "$IPFS_PATH/repo.lock" &&
+  test ! -e "$IPFS_PATH/datastore/LOCK" &&
+  test ! -e "$IPFS_PATH/api"
+'
+
+##########################
+# Test with daemon running
+########################## 
+
+test_launch_ipfs_daemon
+
+# Daemon is running -> command doesn't run
+test_expect_success "'ipfs repo fsck' fails with daemon running" '
+  ! (ipfs repo fsck 2>fsck_out_actual8 )
+
+'
+
+test_expect_success "'ipfs repo fsck' output looks good with daemon" '
+  grep "Error: ipfs daemon is running" fsck_out_actual8
+'
+
+test_kill_ipfs_daemon
+
+test_done