Merge pull request #1598 from ipfs/check-for-api

check for API -- WIP

cmd/ipfs/daemon.go

@@ -295,9 +295,16 @@ func serveHTTPApi(req cmds.Request) (error, <-chan error) {
 		return fmt.Errorf("serveHTTPApi: GetConfig() failed: %s", err), nil
 	}
 
-	apiMaddr, err := ma.NewMultiaddr(cfg.Addresses.API)
+	apiAddr, _, err := req.Option(commands.ApiOption).String()
 	if err != nil {
-		return fmt.Errorf("serveHTTPApi: invalid API address: %q (err: %s)", cfg.Addresses.API, err), nil
+		return fmt.Errorf("serveHTTPApi: %s", err), nil
+	}
+	if apiAddr == "" {
+		apiAddr = cfg.Addresses.API
+	}
+	apiMaddr, err := ma.NewMultiaddr(apiAddr)
+	if err != nil {
+		return fmt.Errorf("serveHTTPApi: invalid API address: %q (err: %s)", apiAddr, err), nil
 	}
 
 	apiLis, err := manet.Listen(apiMaddr)
@@ -347,7 +354,11 @@ func serveHTTPApi(req cmds.Request) (error, <-chan error) {
 
 	node, err := req.InvocContext().ConstructNode()
 	if err != nil {
-		return fmt.Errorf("serveHTTPGateway: ConstructNode() failed: %s", err), nil
+		return fmt.Errorf("serveHTTPApi: ConstructNode() failed: %s", err), nil
+	}
+
+	if err := node.Repo.SetAPIAddr(apiAddr); err != nil {
+		return fmt.Errorf("serveHTTPApi: SetAPIAddr() failed: %s", err), nil
 	}
 
 	errc := make(chan error)

cmd/ipfs/main.go

@@ -23,6 +23,8 @@ import (
 	cmdsCli "github.com/ipfs/go-ipfs/commands/cli"
 	cmdsHttp "github.com/ipfs/go-ipfs/commands/http"
 	core "github.com/ipfs/go-ipfs/core"
+	coreCmds "github.com/ipfs/go-ipfs/core/commands"
+	repo "github.com/ipfs/go-ipfs/repo"
 	config "github.com/ipfs/go-ipfs/repo/config"
 	fsrepo "github.com/ipfs/go-ipfs/repo/fsrepo"
 	eventlog "github.com/ipfs/go-ipfs/thirdparty/eventlog"
@@ -32,8 +34,10 @@ import (
 // log is the command logger
 var log = eventlog.Logger("cmd/ipfs")
 
-// signal to output help
-var errHelpRequested = errors.New("Help Requested")
+var (
+	errUnexpectedApiOutput = errors.New("api returned unexpected output")
+	errApiVersionMismatch  = errors.New("api version mismatch")
+)
 
 const (
 	EnvEnableProfiling = "IPFS_PROF"
@@ -295,8 +299,7 @@ func callCommand(ctx context.Context, req cmds.Request, root *cmds.Command, cmd
 		return nil, err
 	}
 
-	log.Debug("looking for running daemon...")
-	useDaemon, err := commandShouldRunOnDaemon(*details, req, root)
+	client, err := commandShouldRunOnDaemon(*details, req, root)
 	if err != nil {
 		return nil, err
 	}
@@ -313,28 +316,13 @@ func callCommand(ctx context.Context, req cmds.Request, root *cmds.Command, cmd
 		}
 	}
 
-	if useDaemon {
-
-		cfg, err := req.InvocContext().GetConfig()
-		if err != nil {
-			return nil, err
-		}
-
-		addr, err := ma.NewMultiaddr(cfg.Addresses.API)
-		if err != nil {
-			return nil, err
-		}
-
-		log.Infof("Executing command on daemon running at %s", addr)
-		_, host, err := manet.DialArgs(addr)
-		if err != nil {
-			return nil, err
-		}
-
-		client := cmdsHttp.NewClient(host)
-
+	if client != nil {
+		log.Debug("Executing command via API")
 		res, err = client.Send(req)
 		if err != nil {
+			if isConnRefused(err) {
+				err = repo.ErrApiNotRunning
+			}
 			return nil, err
 		}
 
@@ -383,48 +371,67 @@ func commandDetails(path []string, root *cmds.Command) (*cmdDetails, error) {
 // commandShouldRunOnDaemon determines, from commmand details, whether a
 // command ought to be executed on an IPFS daemon.
 //
-// It returns true if the command should be executed on a daemon and false if
+// It returns a client if the command should be executed on a daemon and nil if
 // it should be executed on a client. It returns an error if the command must
 // NOT be executed on either.
-func commandShouldRunOnDaemon(details cmdDetails, req cmds.Request, root *cmds.Command) (bool, error) {
+func commandShouldRunOnDaemon(details cmdDetails, req cmds.Request, root *cmds.Command) (cmdsHttp.Client, error) {
 	path := req.Path()
 	// root command.
 	if len(path) < 1 {
-		return false, nil
+		return nil, nil
 	}
 
 	if details.cannotRunOnClient && details.cannotRunOnDaemon {
-		return false, fmt.Errorf("command disabled: %s", path[0])
+		return nil, fmt.Errorf("command disabled: %s", path[0])
 	}
 
 	if details.doesNotUseRepo && details.canRunOnClient() {
-		return false, nil
+		return nil, nil
 	}
 
-	// at this point need to know whether daemon is running. we defer
-	// to this point so that some commands dont open files unnecessarily.
-	daemonLocked, err := fsrepo.LockedByOtherProcess(req.InvocContext().ConfigRoot)
+	// at this point need to know whether api is running. we defer
+	// to this point so that we dont check unnecessarily
+
+	// did user specify an api to use for this command?
+	apiAddrStr, _, err := req.Option(coreCmds.ApiOption).String()
 	if err != nil {
-		return false, err
+		return nil, err
 	}
 
-	if daemonLocked {
+	client, err := getApiClient(req.InvocContext().ConfigRoot, apiAddrStr)
+	if err == repo.ErrApiNotRunning {
+		if apiAddrStr != "" && req.Command() != daemonCmd {
+			// if user SPECIFIED an api, and this cmd is not daemon
+			// we MUST use it. so error out.
+			return nil, err
+		}
 
-		log.Info("a daemon is running...")
+		// ok for api not to be running
+	} else if err != nil { // some other api error
+		return nil, err
+	}
 
+	if client != nil { // daemon is running
 		if details.cannotRunOnDaemon {
-			e := "ipfs daemon is running. please stop it to run this command"
-			return false, cmds.ClientError(e)
+			e := "cannot use API with this command."
+
+			// check if daemon locked. legacy error text, for now.
+			daemonLocked, _ := fsrepo.LockedByOtherProcess(req.InvocContext().ConfigRoot)
+			if daemonLocked {
+				e = "ipfs daemon is running. please stop it to run this command"
+			}
+
+			return nil, cmds.ClientError(e)
 		}
 
-		return true, nil
+		return client, nil
 	}
 
 	if details.cannotRunOnClient {
-		return false, cmds.ClientError("must run on the ipfs daemon")
+		return nil, cmds.ClientError("must run on the ipfs daemon")
 	}
 
-	return false, nil
+	return nil, nil
 }
 
 func isClientError(err error) bool {
@@ -574,3 +581,92 @@ func profileIfEnabled() (func(), error) {
 	}
 	return func() {}, nil
 }
+
+// getApiClient checks the repo, and the given options, checking for
+// a running API service. if there is one, it returns a client.
+// otherwise, it returns errApiNotRunning, or another error.
+func getApiClient(repoPath, apiAddrStr string) (cmdsHttp.Client, error) {
+
+	if apiAddrStr == "" {
+		var err error
+		if apiAddrStr, err = fsrepo.APIAddr(repoPath); err != nil {
+			return nil, err
+		}
+	}
+
+	addr, err := ma.NewMultiaddr(apiAddrStr)
+	if err != nil {
+		return nil, err
+	}
+
+	client, err := apiClientForAddr(addr)
+	if err != nil {
+		return nil, err
+	}
+
+	// make sure the api is actually running.
+	// this is slow, as it might mean an RTT to a remote server.
+	// TODO: optimize some way
+	if err := apiVersionMatches(client); err != nil {
+		return nil, err
+	}
+
+	return client, nil
+}
+
+// apiVersionMatches checks whether the api server is running the
+// same version of go-ipfs. for now, only the exact same version of
+// client + server work. In the future, we should use semver for
+// proper API versioning! \o/
+func apiVersionMatches(client cmdsHttp.Client) (err error) {
+	ver, err := doVersionRequest(client)
+	if err != nil {
+		return err
+	}
+
+	currv := config.CurrentVersionNumber
+	if ver.Version != currv {
+		return fmt.Errorf("%s (%s != %s)", errApiVersionMismatch, ver.Version, currv)
+	}
+	return nil
+}
+
+func doVersionRequest(client cmdsHttp.Client) (*coreCmds.VersionOutput, error) {
+	cmd := coreCmds.VersionCmd
+	optDefs, err := cmd.GetOptions([]string{})
+	if err != nil {
+		return nil, err
+	}
+
+	req, err := cmds.NewRequest([]string{"version"}, nil, nil, nil, cmd, optDefs)
+	if err != nil {
+		return nil, err
+	}
+
+	res, err := client.Send(req)
+	if err != nil {
+		if isConnRefused(err) {
+			err = repo.ErrApiNotRunning
+		}
+		return nil, err
+	}
+
+	ver, ok := res.Output().(*coreCmds.VersionOutput)
+	if !ok {
+		return nil, errUnexpectedApiOutput
+	}
+	return ver, nil
+}
+
+func apiClientForAddr(addr ma.Multiaddr) (cmdsHttp.Client, error) {
+	_, host, err := manet.DialArgs(addr)
+	if err != nil {
+		return nil, err
+	}
+
+	return cmdsHttp.NewClient(host), nil
+}
+
+func isConnRefused(err error) bool {
+	return strings.Contains(err.Error(), "connection refused")
+}

core/commands/root.go

@@ -16,6 +16,10 @@ type TestOutput struct {
 	Bar int
 }
 
+const (
+	ApiOption = "api"
+)
+
 var Root = &cmds.Command{
 	Helptext: cmds.HelpText{
 		Tagline: "global p2p merkle-dag filesystem",
@@ -73,6 +77,7 @@ Use 'ipfs <command> --help' to learn more about each command.
 		cmds.BoolOption("help", "Show the full command help text"),
 		cmds.BoolOption("h", "Show a short version of the command help text"),
 		cmds.BoolOption("local", "L", "Run the command locally, instead of using the daemon"),
+		cmds.StringOption(ApiOption, "Overrides the routing option (dht, supernode)"),
 	},
 }
 

repo/fsrepo/fsrepo.go

@@ -58,6 +58,7 @@ func (err NoRepoError) Error() string {
 const (
 	leveldbDirectory = "datastore"
 	flatfsDirectory  = "blocks"
+	apiFile          = "api"
 )
 
 var (
@@ -285,14 +286,53 @@ func Remove(repoPath string) error {
 // process. If true, then the repo cannot be opened by this process.
 func LockedByOtherProcess(repoPath string) (bool, error) {
 	repoPath = path.Clean(repoPath)
-
-	// TODO replace this with the "api" file
-	// https://github.com/ipfs/specs/tree/master/repo/fs-repo
-
 	// NB: the lock is only held when repos are Open
 	return lockfile.Locked(repoPath)
 }
 
+// APIAddr returns the registered API addr, according to the api file
+// in the fsrepo. This is a concurrent operation, meaning that any
+// process may read this file. modifying this file, therefore, should
+// use "mv" to replace the whole file and avoid interleaved read/writes.
+func APIAddr(repoPath string) (string, error) {
+	repoPath = path.Clean(repoPath)
+	apiFilePath := path.Join(repoPath, apiFile)
+
+	// if there is no file, assume there is no api addr.
+	f, err := os.Open(apiFilePath)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return "", repo.ErrApiNotRunning
+		}
+		return "", err
+	}
+	defer f.Close()
+
+	// read up to 2048 bytes. io.ReadAll is a vulnerability, as
+	// someone could hose the process by putting a massive file there.
+	buf := make([]byte, 2048)
+	n, err := f.Read(buf)
+	if err != nil && err != io.EOF {
+		return "", err
+	}
+
+	s := string(buf[:n])
+	s = strings.TrimSpace(s)
+	return s, nil
+}
+
+// SetAPIAddr writes the API Addr to the /api file.
+func (r *FSRepo) SetAPIAddr(addr string) error {
+	f, err := os.Create(path.Join(r.path, apiFile))
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	_, err = f.WriteString(addr)
+	return err
+}
+
 // openConfig returns an error if the config file is not present.
 func (r *FSRepo) openConfig() error {
 	configFilename, err := config.Filename(r.path)

repo/mock.go

@@ -35,3 +35,5 @@ func (m *Mock) GetConfigKey(key string) (interface{}, error) {
 func (m *Mock) Datastore() ds.ThreadSafeDatastore { return m.D }
 
 func (m *Mock) Close() error { return errTODO }
+
+func (m *Mock) SetAPIAddr(addr string) error { return errTODO }

repo/repo.go

 package repo
 
 import (
+	"errors"
 	"io"
 
 	datastore "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
+
 	config "github.com/ipfs/go-ipfs/repo/config"
 )
 
+var (
+	ErrApiNotRunning = errors.New("api not running")
+)
+
 type Repo interface {
 	Config() *config.Config
 	SetConfig(*config.Config) error
@@ -16,5 +22,8 @@ type Repo interface {
 
 	Datastore() datastore.ThreadSafeDatastore
 
+	// SetAPIAddr sets the API address in the repo.
+	SetAPIAddr(addr string) error
+
 	io.Closer
 }

test/sharness/README.md

@@ -71,7 +71,8 @@ For example:
 test_expect_success ".ipfs/ has been created" '
   test -d ".ipfs" &&
   test -f ".ipfs/config" &&
-  test -d ".ipfs/datastore" ||
+  test -d ".ipfs/datastore" &&
+  test -d ".ipfs/blocks" ||
   test_fsh ls -al .ipfs
 '
 ```

test/sharness/lib/test-lib.sh

@@ -193,7 +193,7 @@ test_config_ipfs_gateway_writable() {
 
 test_launch_ipfs_daemon() {
 
-	args=$1
+	args="$@"
 
 	test_expect_success "'ipfs daemon' succeeds" '
 		ipfs daemon $args >actual_daemon 2>daemon_err &

test/sharness/t0060-daemon.sh

@@ -75,8 +75,9 @@ test_expect_success "ipfs daemon output looks good" '
 test_expect_success ".ipfs/ has been created" '
   test -d ".ipfs" &&
   test -f ".ipfs/config" &&
-  test -d ".ipfs/datastore" ||
-  test_fsh ls .ipfs
+  test -d ".ipfs/datastore" &&
+  test -d ".ipfs/blocks" ||
+  test_fsh ls -al .ipfs
 '
 
 # begin same as in t0010
@@ -102,7 +103,7 @@ test_expect_success "ipfs help output looks good" '
 
 # check transport is encrypted
 
-test_expect_success 'transport should be encrypted' '
+test_expect_success "transport should be encrypted" '
   nc -w 5 localhost 4001 >swarmnc &&
   grep -q "AES-256,AES-128" swarmnc &&
   test_must_fail grep -q "/ipfs/identify" swarmnc ||

test/sharness/t0061-daemon-opts.sh

@@ -11,7 +11,7 @@ test_description="Test daemon command"
 
 test_init_ipfs
 
-test_launch_ipfs_daemon '--unrestricted-api --disable-transport-encryption'
+test_launch_ipfs_daemon --unrestricted-api --disable-transport-encryption
 
 gwyport=$PORT_GWAY
 apiport=$PORT_API

test/sharness/t0062-daemon-api.sh

+#!/bin/sh
+#
+# MIT Licensed; see the LICENSE file in this repository.
+#
+
+test_description="Test daemon command"
+
+. lib/test-lib.sh
+
+test_init_ipfs
+
+differentport=$((PORT_API + 1))
+api_different="/ip4/127.0.0.1/tcp/$differentport"
+api_unreachable="/ip4/127.0.0.1/tcp/1"
+api_fromcfg=$(ipfs config Addresses.API)
+peerid=$(ipfs config Identity.PeerID)
+
+test_client() {
+	args="$@"
+	printf $peerid >expected
+	ipfs $args id -f="<id>" >actual
+	test_cmp expected actual
+}
+
+test_client_must_fail() {
+	args="$@"
+	echo "Error: api not running" >expected_err
+	test_must_fail ipfs $args id -f="<id>" >actual 2>actual_err
+	test_cmp expected_err actual_err
+}
+
+
+# first, test things without daemon, without /api file
+
+test_expect_success "client should work (daemon off, no /api file, no --api)" '
+	test_client
+'
+
+test_expect_success "client --api fromcfg should err (daemon off, no /api file)" '
+	test_client_must_fail --api "$api_fromcfg"
+'
+
+test_expect_success "client --api unreachable should err (daemon off, no /api file)" '
+	test_client_must_fail --api "$api_unreachable"
+'
+
+# then, test things with daemon, with /api file
+
+test_launch_ipfs_daemon
+
+test_expect_success "'ipfs daemon' creates api file" '
+	test -f ".ipfs/api"
+'
+
+test_expect_success "api file looks good" '
+	printf "$ADDR_API" >expected &&
+	test_cmp expected .ipfs/api
+'
+
+test_expect_success "client should work (daemon on, /api file, no --api)" '
+	test_client
+'
+
+test_expect_success "client --api fromcfg should work (daemon used cfg) (daemon, /api file)" '
+	test_client --api "$api_fromcfg"
+'
+
+test_expect_success "client --api unreachable should err (daemon, /api file)" '
+	test_client_must_fail --api "$api_unreachable"
+'
+
+# then, test things without daemon, with /api file
+
+test_kill_ipfs_daemon
+
+test_expect_success "client should work (daemon off, /api file, no --api)" '
+	test_client
+'
+
+test_expect_success "client --api fromcfg should err (daemon off, /api file)" '
+	test_client_must_fail --api "$api_fromcfg"
+'
+
+test_expect_success "client --api unreachable should err (daemon, /api file)" '
+	test_client_must_fail --api "$api_unreachable"
+'
+
+# then, test things with daemon --api $api_different, with /api file
+
+PORT_API=$differentport
+ADDR_API=$api_different
+
+test_launch_ipfs_daemon --api "$ADDR_API"
+
+test_expect_success "'ipfs daemon' --api option works" '
+	printf "$api_different" >expected &&
+	test_cmp expected .ipfs/api
+'
+
+test_expect_success "client should work (daemon on, /api file (different), no --api)" '
+	test_client
+'
+
+test_expect_success "client --api different should work (daemon on, /api file (different))" '
+	test_client --api "$api_different"
+'
+
+test_expect_success "client --api fromcfg should err (daemon on, /api file (different))" '
+	test_client_must_fail --api "$api_fromcfg"
+'
+
+test_expect_success "client --api unreachable should err (daemon, /api file)" '
+	test_client_must_fail --api "$api_unreachable"
+'
+
+# then, test things with daemon off, with /api file, for good measure.
+
+test_kill_ipfs_daemon
+
+test_expect_success "client should work (daemon off, /api file (different), no --api)" '
+	test_client
+'
+
+test_expect_success "client --api different should work (daemon on, /api file (different))" '
+	test_client_must_fail --api "$api_different"
+'
+
+test_expect_success "client --api fromcfg should err (daemon on, /api file (different))" '
+	test_client_must_fail --api "$api_fromcfg"
+'
+
+test_expect_success "client --api unreachable should err (daemon, /api file)" '
+	test_client_must_fail --api "$api_unreachable"
+'
+
+test_done