Merge pull request #2939 from ipfs/feat/auto-migrate

Automatically download and run migrations if needed

cmd/ipfs/daemon.go

@@ -23,6 +23,7 @@ import (
 	"github.com/ipfs/go-ipfs/core/corerouting"
 	nodeMount "github.com/ipfs/go-ipfs/fuse/node"
 	fsrepo "github.com/ipfs/go-ipfs/repo/fsrepo"
+	migrate "github.com/ipfs/go-ipfs/repo/fsrepo/migrations"
 	pstore "gx/ipfs/QmQdnfvZQuhdT93LNc5bos52wAmdr3G2p6G8teLJMEN32P/go-libp2p-peerstore"
 	conn "gx/ipfs/QmVCe3SNMjkcPgnpFhZs719dheq6xE7gJwjzV7aWcUM4Ms/go-libp2p/p2p/net/conn"
 	util "gx/ipfs/QmZNVWh8LLjAavuQ2JXuFmuYH3C11xo988vSgp7UQrTRj1/go-ipfs-util"
@@ -30,18 +31,19 @@ import (
 )
 
 const (
+	adjustFDLimitKwd          = "manage-fdlimit"
+	enableGCKwd               = "enable-gc"
 	initOptionKwd             = "init"
-	routingOptionKwd          = "routing"
-	routingOptionSupernodeKwd = "supernode"
-	mountKwd                  = "mount"
-	writableKwd               = "writable"
 	ipfsMountKwd              = "mount-ipfs"
 	ipnsMountKwd              = "mount-ipns"
-	unrestrictedApiAccessKwd  = "unrestricted-api"
-	unencryptTransportKwd     = "disable-transport-encryption"
-	enableGCKwd               = "enable-gc"
-	adjustFDLimitKwd          = "manage-fdlimit"
+	migrateKwd                = "migrate"
+	mountKwd                  = "mount"
 	offlineKwd                = "offline"
+	routingOptionKwd          = "routing"
+	routingOptionSupernodeKwd = "supernode"
+	unencryptTransportKwd     = "disable-transport-encryption"
+	unrestrictedApiAccessKwd  = "unrestricted-api"
+	writableKwd               = "writable"
 	// apiAddrKwd    = "address-api"
 	// swarmAddrKwd  = "address-swarm"
 )
@@ -139,6 +141,7 @@ Headers.
 		cmds.BoolOption(enableGCKwd, "Enable automatic periodic repo garbage collection").Default(false),
 		cmds.BoolOption(adjustFDLimitKwd, "Check and raise file descriptor limits if needed").Default(true),
 		cmds.BoolOption(offlineKwd, "Run offline. Do not connect to the rest of the network but provide local API.").Default(false),
+		cmds.BoolOption(migrateKwd, "If true, assume yes at the migrate prompt. If false, assume no."),
 
 		// TODO: add way to override addresses. tricky part: updating the config if also --init.
 		// cmds.StringOption(apiAddrKwd, "Address for the daemon rpc API (overrides config)"),
@@ -216,9 +219,36 @@ func daemonFunc(req cmds.Request, res cmds.Response) {
 	// acquire the repo lock _before_ constructing a node. we need to make
 	// sure we are permitted to access the resources (datastore, etc.)
 	repo, err := fsrepo.Open(req.InvocContext().ConfigRoot)
-	if err != nil {
+	switch err {
+	default:
 		res.SetError(err, cmds.ErrNormal)
 		return
+	case fsrepo.ErrNeedMigration:
+		domigrate, found, _ := req.Option(migrateKwd).Bool()
+		fmt.Println("Found old repo version, migrations need to be run.")
+
+		if !found {
+			domigrate = YesNoPrompt("Run migrations automatically? [y/N]")
+		}
+
+		if !domigrate {
+			res.SetError(fmt.Errorf("please run the migrations manually"), cmds.ErrNormal)
+			return
+		}
+
+		err = migrate.RunMigration(fsrepo.RepoVersion)
+		if err != nil {
+			res.SetError(err, cmds.ErrNormal)
+			return
+		}
+
+		repo, err = fsrepo.Open(req.InvocContext().ConfigRoot)
+		if err != nil {
+			res.SetError(err, cmds.ErrNormal)
+			return
+		}
+	case nil:
+		break
 	}
 
 	cfg, err := ctx.GetConfig()
@@ -569,3 +599,22 @@ func merge(cs ...<-chan error) <-chan error {
 	}()
 	return out
 }
+
+func YesNoPrompt(prompt string) bool {
+	var s string
+	for i := 0; i < 3; i++ {
+		fmt.Printf("%s ", prompt)
+		fmt.Scanf("%s", &s)
+		switch s {
+		case "y", "Y":
+			return true
+		case "n", "N":
+			return false
+		case "":
+			return false
+		}
+		fmt.Println("Please press either 'y' or 'n'")
+	}
+
+	return false
+}

core/commands/repo.go

@@ -320,7 +320,7 @@ var repoVersionCmd = &cmds.Command{
 	},
 	Run: func(req cmds.Request, res cmds.Response) {
 		res.SetOutput(&RepoVersion{
-			Version: fsrepo.RepoVersion,
+			Version: fmt.Sprint(fsrepo.RepoVersion),
 		})
 	},
 	Type: RepoVersion{},

core/commands/version.go

@@ -35,7 +35,7 @@ var VersionCmd = &cmds.Command{
 		res.SetOutput(&VersionOutput{
 			Version: config.CurrentVersionNumber,
 			Commit:  config.CurrentCommit,
-			Repo:    fsrepo.RepoVersion,
+			Repo:    fmt.Sprint(fsrepo.RepoVersion),
 			System:  runtime.GOARCH + "/" + runtime.GOOS, //TODO: Precise version here
 			Golang:  runtime.Version(),
 		})

core/corerepo/stat.go

 package corerepo
 
 import (
+	"fmt"
+
 	"github.com/ipfs/go-ipfs/core"
 	fsrepo "github.com/ipfs/go-ipfs/repo/fsrepo"
 	context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context"
@@ -40,6 +42,6 @@ func RepoStat(n *core.IpfsNode, ctx context.Context) (*Stat, error) {
 		NumObjects: count,
 		RepoSize:   usage,
 		RepoPath:   path,
-		Version:    "fs-repo@" + fsrepo.RepoVersion,
+		Version:    fmt.Sprintf("fs-repo@%d", fsrepo.RepoVersion),
 	}, nil
 }

repo/fsrepo/fsrepo.go

@@ -26,7 +26,7 @@ import (
 var log = logging.Logger("fsrepo")
 
 // version number that we are currently expecting to see
-var RepoVersion = "4"
+var RepoVersion = 4
 
 var migrationInstructions = `See https://github.com/ipfs/fs-repo-migrations/blob/master/run.md
 Sorry for the inconvenience. In the future, these will run automatically.`
@@ -36,9 +36,16 @@ Program version is: %s
 Please run the ipfs migration tool before continuing.
 ` + migrationInstructions
 
+var programTooLowMessage = `Your programs version (%d) is lower than your repos (%d).
+Please update ipfs to a version that supports the existing repo, or run
+a migration in reverse.
+
+See https://github.com/ipfs/fs-repo-migrations/blob/master/run.md for details.`
+
 var (
-	ErrNoVersion = errors.New("no version file found, please run 0-to-1 migration tool.\n" + migrationInstructions)
-	ErrOldRepo   = errors.New("ipfs repo found in old '~/.go-ipfs' location, please run migration tool.\n" + migrationInstructions)
+	ErrNoVersion     = errors.New("no version file found, please run 0-to-1 migration tool.\n" + migrationInstructions)
+	ErrOldRepo       = errors.New("ipfs repo found in old '~/.go-ipfs' location, please run migration tool.\n" + migrationInstructions)
+	ErrNeedMigration = errors.New("ipfs repo needs migration.")
 )
 
 type NoRepoError struct {
@@ -134,8 +141,11 @@ func open(repoPath string) (repo.Repo, error) {
 		return nil, err
 	}
 
-	if ver != RepoVersion {
-		return nil, fmt.Errorf(errIncorrectRepoFmt, ver, RepoVersion)
+	if RepoVersion > ver {
+		return nil, ErrNeedMigration
+	} else if ver > RepoVersion {
+		// program version too low for existing repo
+		return nil, fmt.Errorf(programTooLowMessage, RepoVersion, ver)
 	}
 
 	// check repo path, then check all constituent parts.

repo/fsrepo/migrations/mfsr.go

@@ -5,6 +5,7 @@ import (
 	"io/ioutil"
 	"os"
 	"path"
+	"strconv"
 	"strings"
 )
 
@@ -16,27 +17,26 @@ func (rp RepoPath) VersionFile() string {
 	return path.Join(string(rp), VersionFile)
 }
 
-func (rp RepoPath) Version() (string, error) {
+func (rp RepoPath) Version() (int, error) {
 	if rp == "" {
-		return "", fmt.Errorf("invalid repo path \"%s\"", rp)
+		return 0, fmt.Errorf("invalid repo path \"%s\"", rp)
 	}
 
 	fn := rp.VersionFile()
 	if _, err := os.Stat(fn); os.IsNotExist(err) {
-		return "", VersionFileNotFound(rp)
+		return 0, VersionFileNotFound(rp)
 	}
 
 	c, err := ioutil.ReadFile(fn)
 	if err != nil {
-		return "", err
+		return 0, err
 	}
 
-	s := string(c)
-	s = strings.TrimSpace(s)
-	return s, nil
+	s := strings.TrimSpace(string(c))
+	return strconv.Atoi(s)
 }
 
-func (rp RepoPath) CheckVersion(version string) error {
+func (rp RepoPath) CheckVersion(version int) error {
 	v, err := rp.Version()
 	if err != nil {
 		return err
@@ -49,9 +49,9 @@ func (rp RepoPath) CheckVersion(version string) error {
 	return nil
 }
 
-func (rp RepoPath) WriteVersion(version string) error {
+func (rp RepoPath) WriteVersion(version int) error {
 	fn := rp.VersionFile()
-	return ioutil.WriteFile(fn, []byte(version+"\n"), 0644)
+	return ioutil.WriteFile(fn, []byte(fmt.Sprintf("%d\n", version)), 0644)
 }
 
 type VersionFileNotFound string

repo/fsrepo/migrations/migrations.go

+package mfsr
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"strconv"
+	"strings"
+)
+
+var DistPath = "https://ipfs.io/ipfs/QmUnvqDuRyfe7HJuiMMHv77AMUFnjGyAU28LFPeTYwGmFF"
+
+func init() {
+	if dist := os.Getenv("IPFS_DIST_PATH"); dist != "" {
+		DistPath = dist
+	}
+}
+
+const migrations = "fs-repo-migrations"
+
+func RunMigration(newv int) error {
+	migrateBin := "fs-repo-migrations"
+	fmt.Println("  => checking for migrations binary...")
+
+	var err error
+	migrateBin, err = exec.LookPath(migrateBin)
+	if err == nil {
+		// check to make sure migrations binary supports our target version
+		err = verifyMigrationSupportsVersion(migrateBin, newv)
+	}
+
+	if err != nil {
+		fmt.Println("  => usable migrations not found on system, fetching...")
+		loc, err := GetMigrations()
+		if err != nil {
+			return err
+		}
+
+		err = verifyMigrationSupportsVersion(loc, newv)
+		if err != nil {
+			return fmt.Errorf("no migration binary found that supports version %d - %s", newv, err)
+		}
+
+		migrateBin = loc
+	}
+
+	cmd := exec.Command(migrateBin, "-to", fmt.Sprint(newv), "-y")
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+
+	fmt.Printf("  => running migration: '%s -to %d -y'\n\n", migrateBin, newv)
+
+	err = cmd.Run()
+	if err != nil {
+		return fmt.Errorf("migration failed: %s", err)
+	}
+
+	fmt.Println("  => migrations binary completed successfully")
+
+	return nil
+}
+
+func GetMigrations() (string, error) {
+	latest, err := GetLatestVersion(DistPath, migrations)
+	if err != nil {
+		return "", fmt.Errorf("getting latest version of fs-repo-migrations: %s", err)
+	}
+
+	dir, err := ioutil.TempDir("", "go-ipfs-migrate")
+	if err != nil {
+		return "", fmt.Errorf("tempdir: %s", err)
+	}
+
+	out := filepath.Join(dir, migrations)
+
+	err = GetBinaryForVersion(migrations, migrations, DistPath, latest, out)
+	if err != nil {
+		fmt.Printf("  => error getting migrations binary: %s\n", err)
+		fmt.Println("  => could not find or install fs-repo-migrations, please manually install it")
+		return "", fmt.Errorf("failed to find migrations binary")
+	}
+
+	err = os.Chmod(out, 0755)
+	if err != nil {
+		return "", err
+	}
+
+	return out, nil
+}
+
+func verifyMigrationSupportsVersion(fsrbin string, vn int) error {
+	sn, err := migrationsVersion(fsrbin)
+	if err != nil {
+		return err
+	}
+
+	if sn >= vn {
+		return nil
+	}
+
+	return fmt.Errorf("migrations binary doesnt support version %d: %s", vn, fsrbin)
+}
+
+func migrationsVersion(bin string) (int, error) {
+	out, err := exec.Command(bin, "-v").CombinedOutput()
+	if err != nil {
+		return 0, fmt.Errorf("failed to check migrations version: %s", err)
+	}
+
+	vs := strings.Trim(string(out), " \n\t")
+	vn, err := strconv.Atoi(vs)
+	if err != nil {
+		return 0, fmt.Errorf("migrations binary version check did not return a number")
+	}
+
+	return vn, nil
+}
+
+func GetVersions(ipfspath, dist string) ([]string, error) {
+	rc, err := httpFetch(ipfspath + "/" + dist + "/versions")
+	if err != nil {
+		return nil, err
+	}
+	defer rc.Close()
+
+	var out []string
+	scan := bufio.NewScanner(rc)
+	for scan.Scan() {
+		out = append(out, scan.Text())
+	}
+
+	return out, nil
+}
+
+func GetLatestVersion(ipfspath, dist string) (string, error) {
+	vs, err := GetVersions(ipfspath, dist)
+	if err != nil {
+		return "", err
+	}
+	var latest string
+	for i := len(vs) - 1; i >= 0; i-- {
+		if !strings.Contains(vs[i], "-dev") {
+			latest = vs[i]
+			break
+		}
+	}
+	if latest == "" {
+		return "", fmt.Errorf("couldnt find a non dev version in the list")
+	}
+	return vs[len(vs)-1], nil
+}
+
+func httpGet(url string) (*http.Response, error) {
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return nil, fmt.Errorf("http.NewRequest error: %s", err)
+	}
+
+	req.Header.Set("User-Agent", "go-ipfs")
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("http.DefaultClient.Do error: %s", err)
+	}
+
+	return resp, nil
+}
+
+func httpFetch(url string) (io.ReadCloser, error) {
+	fmt.Printf("fetching url: %s\n", url)
+	resp, err := httpGet(url)
+	if err != nil {
+		return nil, err
+	}
+
+	if resp.StatusCode >= 400 {
+		mes, err := ioutil.ReadAll(resp.Body)
+		if err != nil {
+			return nil, fmt.Errorf("error reading error body: %s", err)
+		}
+
+		return nil, fmt.Errorf("%s: %s", resp.Status, string(mes))
+	}
+
+	return resp.Body, nil
+}
+
+func GetBinaryForVersion(distname, binnom, root, vers, out string) error {
+	dir, err := ioutil.TempDir("", "go-ipfs-auto-migrate")
+	if err != nil {
+		return err
+	}
+
+	var archive string
+	switch runtime.GOOS {
+	case "windows":
+		archive = "zip"
+	default:
+		archive = "tar.gz"
+	}
+	osv, err := osWithVariant()
+	if err != nil {
+		return err
+	}
+	finame := fmt.Sprintf("%s_%s_%s-%s.%s", distname, vers, osv, runtime.GOARCH, archive)
+	distpath := fmt.Sprintf("%s/%s/%s/%s", root, distname, vers, finame)
+
+	data, err := httpFetch(distpath)
+	if err != nil {
+		return err
+	}
+
+	arcpath := filepath.Join(dir, finame)
+	fi, err := os.Create(arcpath)
+	if err != nil {
+		return err
+	}
+
+	_, err = io.Copy(fi, data)
+	if err != nil {
+		return err
+	}
+	fi.Close()
+
+	return unpackArchive(distname, binnom, arcpath, out, archive)
+}
+
+func osWithVariant() (string, error) {
+	if runtime.GOOS != "linux" {
+		return runtime.GOOS, nil
+	}
+
+	bin, err := exec.LookPath(filepath.Base(os.Args[0]))
+	if err != nil {
+		return "", fmt.Errorf("failed to resolve go-ipfs: %s", err)
+	}
+
+	cmd := exec.Command("ldd", bin)
+	buf := new(bytes.Buffer)
+	cmd.Stdout = buf
+	err = cmd.Run()
+	if err != nil {
+		return "", fmt.Errorf("failed to run ldd: %s", err)
+	}
+
+	scan := bufio.NewScanner(buf)
+	for scan.Scan() {
+		if strings.Contains(scan.Text(), "libc") && strings.Contains(scan.Text(), "musl") {
+			return "linux-musl", nil
+		}
+	}
+
+	return "linux", nil
+}

repo/fsrepo/migrations/unpack.go

+package mfsr
+
+import (
+	"archive/tar"
+	"archive/zip"
+	"compress/gzip"
+	"fmt"
+	"io"
+	"os"
+)
+
+func unpackArchive(dist, binnom, path, out, atype string) error {
+	switch atype {
+	case "zip":
+		return unpackZip(dist, binnom, path, out)
+	case "tar.gz":
+		return unpackTgz(dist, binnom, path, out)
+	default:
+		return fmt.Errorf("unrecognized archive type: %s", atype)
+	}
+}
+
+func unpackTgz(dist, binnom, path, out string) error {
+	fi, err := os.Open(path)
+	if err != nil {
+		return err
+	}
+	defer fi.Close()
+
+	gzr, err := gzip.NewReader(fi)
+	if err != nil {
+		return err
+	}
+
+	defer gzr.Close()
+
+	var bin io.Reader
+	tarr := tar.NewReader(gzr)
+
+loop:
+	for {
+		th, err := tarr.Next()
+		switch err {
+		default:
+			return err
+		case io.EOF:
+			break loop
+		case nil:
+			// continue
+		}
+
+		if th.Name == dist+"/"+binnom {
+			bin = tarr
+			break
+		}
+	}
+
+	if bin == nil {
+		return fmt.Errorf("no binary found in downloaded archive")
+	}
+
+	return writeToPath(bin, out)
+}
+
+func writeToPath(rc io.Reader, out string) error {
+	binfi, err := os.Create(out)
+	if err != nil {
+		return fmt.Errorf("error opening tmp bin path '%s': %s", out, err)
+	}
+	defer binfi.Close()
+
+	_, err = io.Copy(binfi, rc)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func unpackZip(dist, binnom, path, out string) error {
+	zipr, err := zip.OpenReader(path)
+	if err != nil {
+		return fmt.Errorf("error opening zipreader: %s", err)
+	}
+
+	defer zipr.Close()
+
+	var bin io.ReadCloser
+	for _, fis := range zipr.File {
+		if fis.Name == dist+"/"+binnom+".exe" {
+			rc, err := fis.Open()
+			if err != nil {
+				return fmt.Errorf("error extracting binary from archive: %s", err)
+			}
+
+			bin = rc
+		}
+	}
+
+	return writeToPath(bin, out)
+}

repo/mock.go

@@ -6,7 +6,7 @@ import (
 	"github.com/ipfs/go-ipfs/repo/config"
 )
 
-var errTODO = errors.New("TODO")
+var errTODO = errors.New("TODO: mock repo")
 
 // Mock is not thread-safe
 type Mock struct {

test/sharness/t0066-migration.sh

+#!/bin/sh
+#
+# Copyright (c) 2016 Jeromy Johnson
+# MIT Licensed; see the LICENSE file in this repository.
+#
+
+test_description="Test migrations auto update prompt"
+
+. lib/test-lib.sh
+
+test_init_ipfs
+
+test_expect_success "setup mock migrations" '
+	mkdir bin &&
+	echo "#!/bin/bash" > bin/fs-repo-migrations &&
+	echo "echo 4" >> bin/fs-repo-migrations &&
+	chmod +x bin/fs-repo-migrations &&
+	export PATH="$(pwd)/bin":$PATH
+'
+
+test_expect_success "manually reset repo version to 3" '
+	echo "3" > "$IPFS_PATH"/version
+'
+
+test_expect_success "ipfs daemon --migrate=false fails" '
+	test_expect_code 1 ipfs daemon --migrate=false 2> false_out
+'
+
+test_expect_success "output looks good" '
+	grep "please run the migrations manually" false_out
+'
+
+test_expect_success "ipfs daemon --migrate=true runs migration" '
+	test_expect_code 1 ipfs daemon --migrate=true > true_out
+'
+
+test_expect_success "output looks good" '
+	grep "running migration" true_out > /dev/null &&
+	grep "binary completed successfully" true_out > /dev/null
+'
+
+test_expect_success "'ipfs daemon' prompts to auto migrate" '
+	test_expect_code 1 ipfs daemon > daemon_out 2> daemon_err
+'
+
+test_expect_success "output looks good" '
+	grep "Found old repo version" daemon_out > /dev/null &&
+	grep "Run migrations automatically?" daemon_out > /dev/null &&
+	grep "please run the migrations manually" daemon_err > /dev/null
+'
+
+test_done