Merge pull request #1114 from ipfs/test/3node-sharness

add clone of 3node test using iptb instead of docker

Godeps/Godeps.json

@@ -37,6 +37,10 @@
 			"Rev": "068b72961a6bc5b4a82cf4fc14ccc724c0cfa73a"
 		},
 		{
+			"ImportPath":"github.com/whyrusleeping/iptb",
+			"Rev": "5ee5bc0bb43502dfc798786a78df2448c91dd764"
+		},
+		{
 			"ImportPath": "github.com/Sirupsen/logrus",
 			"Comment": "v0.7.1",
 			"Rev": "3fc34d061b9c78a70db853c7cb6b0576b6d4f32d"

Godeps/_workspace/src/github.com/whyrusleeping/iptb/LICENSE

+The MIT License (MIT)
+
+Copyright (c) 2015 Jeromy Johnson
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

Godeps/_workspace/src/github.com/whyrusleeping/iptb/README.md

+#Ipfs Testbed
+
+##commands:
+
+### init -n=[number of nodes]
+creates and initializes 'n' repos
+
+### start 
+starts up all testbed nodes
+
+### stop 
+kills all testbed nodes
+
+### restart
+kills, then restarts all testbed nodes
+
+### shell [n]
+execs your shell with environment variables set as follows:
+- IPFS_PATH - set to testbed node n's IPFS_PATH
+- NODE[x] - set to the peer ID of node x

Godeps/_workspace/src/github.com/whyrusleeping/iptb/main.go

+package main
+
+import (
+	"flag"
+	"fmt"
+	serial "github.com/ipfs/go-ipfs/repo/fsrepo/serialize"
+	"io/ioutil"
+	"log"
+	"net"
+	"os"
+	"os/exec"
+	"path"
+	"strconv"
+	"sync"
+	"syscall"
+	"time"
+)
+
+// GetNumNodes returns the number of testbed nodes configured in the testbed directory
+func GetNumNodes() int {
+	for i := 0; i < 2000; i++ {
+		_, err := os.Stat(IpfsDirN(i))
+		if os.IsNotExist(err) {
+			return i
+		}
+	}
+	panic("i dont know whats going on")
+}
+
+func TestBedDir() string {
+	tbd := os.Getenv("IPTB_ROOT")
+	if len(tbd) != 0 {
+		return tbd
+	}
+
+	home := os.Getenv("HOME")
+	if len(home) == 0 {
+		panic("could not find home")
+	}
+
+	return path.Join(home, "testbed")
+}
+
+func IpfsDirN(n int) string {
+	return path.Join(TestBedDir(), fmt.Sprint(n))
+}
+
+func YesNoPrompt(prompt string) bool {
+	var s string
+	for {
+		fmt.Println(prompt)
+		fmt.Scanf("%s", &s)
+		switch s {
+		case "y", "Y":
+			return true
+		case "n", "N":
+			return false
+		}
+		fmt.Println("Please press either 'y' or 'n'")
+	}
+}
+
+type initCfg struct {
+	Count     int
+	Force     bool
+	Bootstrap string
+}
+
+func IpfsInit(cfg *initCfg) error {
+	p := IpfsDirN(0)
+	if _, err := os.Stat(p); !os.IsNotExist(err) {
+		if !cfg.Force && !YesNoPrompt("testbed nodes already exist, overwrite? [y/n]") {
+			return nil
+		}
+		err := os.RemoveAll(TestBedDir())
+		if err != nil {
+			return err
+		}
+	}
+	wait := sync.WaitGroup{}
+	for i := 0; i < cfg.Count; i++ {
+		wait.Add(1)
+		go func(v int) {
+			defer wait.Done()
+			dir := IpfsDirN(v)
+			err := os.MkdirAll(dir, 0777)
+			if err != nil {
+				log.Println("ERROR: ", err)
+				return
+			}
+
+			cmd := exec.Command("ipfs", "init", "-b=1024")
+			cmd.Env = append(cmd.Env, "IPFS_PATH="+dir)
+			out, err := cmd.CombinedOutput()
+			if err != nil {
+				log.Println("ERROR: ", err)
+				log.Println(string(out))
+			}
+		}(i)
+	}
+	wait.Wait()
+
+	// Now setup bootstrapping
+	switch cfg.Bootstrap {
+	case "star":
+		err := starBootstrap(cfg)
+		if err != nil {
+			return err
+		}
+	case "none":
+		err := clearBootstrapping(cfg)
+		if err != nil {
+			return err
+		}
+	default:
+		return fmt.Errorf("unrecognized bootstrapping option: %s", cfg.Bootstrap)
+	}
+
+	return nil
+}
+
+func starBootstrap(cfg *initCfg) error {
+	// '0' node is the bootstrap node
+	cfgpath := path.Join(IpfsDirN(0), "config")
+	bcfg, err := serial.Load(cfgpath)
+	if err != nil {
+		return err
+	}
+	bcfg.Bootstrap = nil
+	bcfg.Addresses.Swarm = []string{"/ip4/127.0.0.1/tcp/4002"}
+	bcfg.Addresses.API = "/ip4/127.0.0.1/tcp/5002"
+	bcfg.Addresses.Gateway = ""
+	err = serial.WriteConfigFile(cfgpath, bcfg)
+	if err != nil {
+		return err
+	}
+
+	for i := 1; i < cfg.Count; i++ {
+		cfgpath := path.Join(IpfsDirN(i), "config")
+		cfg, err := serial.Load(cfgpath)
+		if err != nil {
+			return err
+		}
+
+		cfg.Bootstrap = []string{fmt.Sprintf("%s/ipfs/%s", bcfg.Addresses.Swarm[0], bcfg.Identity.PeerID)}
+		cfg.Addresses.Gateway = ""
+		cfg.Addresses.Swarm = []string{
+			fmt.Sprintf("/ip4/0.0.0.0/tcp/%d", 4002+i),
+		}
+		cfg.Addresses.API = fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", 5002+i)
+		err = serial.WriteConfigFile(cfgpath, cfg)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func clearBootstrapping(cfg *initCfg) error {
+	for i := 0; i < cfg.Count; i++ {
+		cfgpath := path.Join(IpfsDirN(i), "config")
+		cfg, err := serial.Load(cfgpath)
+		if err != nil {
+			return err
+		}
+
+		cfg.Bootstrap = nil
+		cfg.Addresses.Gateway = ""
+		cfg.Addresses.Swarm = []string{
+			fmt.Sprintf("/ip4/0.0.0.0/tcp/%d", 4002+i),
+		}
+		cfg.Addresses.API = fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", 5002+i)
+		err = serial.WriteConfigFile(cfgpath, cfg)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func IpfsPidOf(n int) (int, error) {
+	dir := IpfsDirN(n)
+	b, err := ioutil.ReadFile(path.Join(dir, "daemon.pid"))
+	if err != nil {
+		return -1, err
+	}
+
+	return strconv.Atoi(string(b))
+}
+
+func IpfsKill() error {
+	n := GetNumNodes()
+	for i := 0; i < n; i++ {
+		pid, err := IpfsPidOf(i)
+		if err != nil {
+			fmt.Printf("error killing daemon %d: %s\n", i, err)
+			continue
+		}
+
+		p, err := os.FindProcess(pid)
+		if err != nil {
+			fmt.Printf("error killing daemon %d: %s\n", i, err)
+			continue
+		}
+		err = p.Kill()
+		if err != nil {
+			fmt.Printf("error killing daemon %d: %s\n", i, err)
+			continue
+		}
+
+		p.Wait()
+
+		err = os.Remove(path.Join(IpfsDirN(i), "daemon.pid"))
+		if err != nil {
+			fmt.Printf("error removing pid file for daemon %d: %s\n", i, err)
+			continue
+		}
+	}
+	return nil
+}
+
+func IpfsStart(waitall bool) error {
+	n := GetNumNodes()
+	for i := 0; i < n; i++ {
+		dir := IpfsDirN(i)
+		cmd := exec.Command("ipfs", "daemon")
+		cmd.Dir = dir
+		cmd.Env = []string{"IPFS_PATH=" + dir}
+
+		cmd.SysProcAttr = &syscall.SysProcAttr{Setsid: true}
+
+		stdout, err := os.Create(path.Join(dir, "daemon.stdout"))
+		if err != nil {
+			return err
+		}
+
+		stderr, err := os.Create(path.Join(dir, "daemon.stderr"))
+		if err != nil {
+			return err
+		}
+
+		cmd.Stdout = stdout
+		cmd.Stderr = stderr
+
+		err = cmd.Start()
+		if err != nil {
+			return err
+		}
+		pid := cmd.Process.Pid
+
+		fmt.Printf("Started daemon %d, pid = %d\n", i, pid)
+		err = ioutil.WriteFile(path.Join(dir, "daemon.pid"), []byte(fmt.Sprint(pid)), 0666)
+		if err != nil {
+			return err
+		}
+
+		// Make sure node 0 is up before starting the rest so
+		// bootstrapping works properly
+		if i == 0 || waitall {
+			err := waitForLive(fmt.Sprintf("localhost:%d", 5002+i))
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+// waitForLive polls the given endpoint until it is up, or until
+// a timeout
+func waitForLive(addr string) error {
+	for i := 0; i < 50; i++ {
+		c, err := net.Dial("tcp", addr)
+		if err == nil {
+			c.Close()
+			return nil
+		}
+		time.Sleep(time.Millisecond * 200)
+	}
+	return fmt.Errorf("node at %s failed to come online in given time period", addr)
+}
+
+// GetPeerID reads the config of node 'n' and returns its peer ID
+func GetPeerID(n int) (string, error) {
+	cfg, err := serial.Load(path.Join(IpfsDirN(n), "config"))
+	if err != nil {
+		return "", err
+	}
+	return cfg.Identity.PeerID, nil
+}
+
+// IpfsShell sets up environment variables for a new shell to more easily
+// control the given daemon
+func IpfsShell(n int) error {
+	shell := os.Getenv("SHELL")
+	if shell == "" {
+		return fmt.Errorf("couldnt find shell!")
+	}
+
+	dir := IpfsDirN(n)
+	nenvs := []string{"IPFS_PATH=" + dir}
+
+	nnodes := GetNumNodes()
+	for i := 0; i < nnodes; i++ {
+		peerid, err := GetPeerID(i)
+		if err != nil {
+			return err
+		}
+		nenvs = append(nenvs, fmt.Sprintf("NODE%d=%s", i, peerid))
+	}
+	nenvs = append(os.Environ(), nenvs...)
+
+	return syscall.Exec(shell, []string{shell}, nenvs)
+}
+
+var helptext = `Ipfs Testbed
+
+Commands:
+	init 
+	    creates and initializes 'n' repos
+
+		Options:
+			-n=[number of nodes]
+			-f - force overwriting of existing nodes
+			-bootstrap - select bootstrapping style for cluster
+				choices: star, none
+
+	start 
+	    starts up all testbed nodes
+
+		Options:
+			-wait - wait until daemons are fully initialized
+	stop 
+	    kills all testbed nodes
+	restart
+	    kills, then restarts all testbed nodes
+
+	shell [n]
+	    execs your shell with environment variables set as follows:
+	        IPFS_PATH - set to testbed node n's IPFS_PATH
+	        NODE[x] - set to the peer ID of node x
+
+Env Vars:
+
+IPTB_ROOT:
+	Used to specify the directory that nodes will be created in.
+`
+
+func handleErr(s string, err error) {
+	if err != nil {
+		fmt.Println(s, err)
+		os.Exit(1)
+	}
+}
+
+func main() {
+	cfg := new(initCfg)
+	flag.IntVar(&cfg.Count, "n", 0, "number of ipfs nodes to initialize")
+	flag.BoolVar(&cfg.Force, "f", false, "force initialization (overwrite existing configs)")
+	flag.StringVar(&cfg.Bootstrap, "bootstrap", "star", "select bootstrapping style for cluster")
+
+	wait := flag.Bool("wait", false, "wait for nodes to come fully online before exiting")
+	flag.Usage = func() {
+		fmt.Println(helptext)
+	}
+
+	flag.Parse()
+
+	switch flag.Arg(0) {
+	case "init":
+		if cfg.Count == 0 {
+			fmt.Printf("please specify number of nodes: '%s -n=10 init'\n", os.Args[0])
+			os.Exit(1)
+		}
+		err := IpfsInit(cfg)
+		handleErr("ipfs init err: ", err)
+	case "start":
+		err := IpfsStart(*wait)
+		handleErr("ipfs start err: ", err)
+	case "stop", "kill":
+		err := IpfsKill()
+		handleErr("ipfs kill err: ", err)
+	case "restart":
+		err := IpfsKill()
+		handleErr("ipfs kill err: ", err)
+
+		err = IpfsStart(*wait)
+		handleErr("ipfs start err: ", err)
+	case "shell":
+		if len(flag.Args()) < 2 {
+			fmt.Println("please specify which node you want a shell for")
+			os.Exit(1)
+		}
+		n, err := strconv.Atoi(flag.Arg(1))
+		handleErr("parse err: ", err)
+
+		err = IpfsShell(n)
+		handleErr("ipfs shell err: ", err)
+	default:
+		flag.Usage()
+		os.Exit(1)
+	}
+}

test/Makefile

@@ -4,6 +4,7 @@ IPFS_ROOT = ../
 IPFS_CMD = ../cmd/ipfs
 RANDOM_SRC = ../Godeps/_workspace/src/github.com/jbenet/go-random
 MULTIHASH_SRC = ../Godeps/_workspace/src/github.com/jbenet/go-multihash
+IPTB_SRC = ../Godeps/_workspace/src/github.com/whyrusleeping/iptb
 POLLENDPOINT_SRC= ../thirdparty/pollEndpoint
 
 # User might want to override those on the command line
@@ -36,6 +37,10 @@ bin/pollEndpoint: $(call find_go_files, $(POLLENDPOINT_SRC)) IPFS-BUILD-OPTIONS
 	@echo "*** installing $@ ***"
 	go build $(GOFLAGS) -o bin/pollEndpoint $(POLLENDPOINT_SRC)
 
+bin/iptb: $(call find_go_files, $(IPTB_SRC)) IPFS-BUILD-OPTIONS
+	@echo "*** installing $@ ***"
+	go build $(GOFLAGS) -o bin/iptb $(IPTB_SRC)
+
 test: test_expensive
 
 test_expensive:

test/sharness/Makefile

@@ -7,7 +7,8 @@
 # NOTE: run with TEST_VERBOSE=1 for verbose sharness tests.
 
 T = $(sort $(wildcard t[0-9][0-9][0-9][0-9]-*.sh))
-BINS = bin/random bin/multihash bin/ipfs bin/pollEndpoint
+BINS = bin/random bin/multihash bin/ipfs bin/pollEndpoint \
+	   bin/iptb
 SHARNESS = lib/sharness/sharness.sh
 IPFS_ROOT = ../..
 

test/sharness/t0130-multinode.sh

+#!/bin/sh
+#
+# Copyright (c) 2015 Jeromy Johnson
+# MIT Licensed; see the LICENSE file in this repository.
+#
+
+test_description="Test multiple ipfs nodes"
+
+. lib/test-lib.sh
+
+export IPTB_ROOT="`pwd`/.iptb"
+
+test_expect_success "set up a few nodes" '
+	iptb -n=3 init &&
+	iptb -wait start
+'
+
+test_expect_success "add a file on node1" '
+	export IPFS_PATH="$IPTB_ROOT/1"
+	random 1000000 > filea &&
+	FILEA_HASH=`ipfs add -q filea`
+'
+
+test_expect_success "cat that file on node2" '
+	export IPFS_PATH="$IPTB_ROOT/2"
+	ipfs cat $FILEA_HASH | multihash > actual1
+'
+
+test_expect_success "verify files match" '
+	multihash filea > expected1 &&
+	test_cmp actual1 expected1
+'
+
+test_expect_success "shut down nodes" '
+	iptb stop
+'
+
+test_done