Add config option for flatfs no-sync

License: MIT
Signed-off-by: rht <rhtbot@gmail.com>

Godeps/Godeps.json

@@ -166,7 +166,7 @@
 		},
 		{
 			"ImportPath": "github.com/jbenet/go-datastore",
-			"Rev": "c835c30f206c1e97172e428f052e225adab9abde"
+			"Rev": "bec407bccea1cfaf56ee946e947642e3ac5a9258"
 		},
 		{
 			"ImportPath": "github.com/jbenet/go-detect-race",

Godeps/_workspace/src/github.com/jbenet/go-datastore/coalesce/coalesce.go

@@ -8,10 +8,10 @@ import (
 	dsq "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/query"
 )
 
+// parent keys
 var (
-	putKey = "put"
-	getKey = // parent keys
-	"get"
+	putKey    = "put"
+	getKey    = "get"
 	hasKey    = "has"
 	deleteKey = "delete"
 )

Godeps/_workspace/src/github.com/jbenet/go-datastore/elastigo/datastore.go

@@ -6,9 +6,10 @@ import (
 	"net/url"
 	"strings"
 
-	"github.com/codahale/blake2"
 	ds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
 	query "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/query"
+
+	"github.com/codahale/blake2"
 	"github.com/mattbaird/elastigo/api"
 	"github.com/mattbaird/elastigo/core"
 )

Godeps/_workspace/src/github.com/jbenet/go-datastore/flatfs/flatfs.go

@@ -15,6 +15,7 @@ import (
 	"github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
 	"github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/query"
 	"github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-os-rename"
+
 	logging "github.com/ipfs/go-ipfs/vendor/QmQg1J6vikuXF9oDvm4wpdeAUvvkVEKW1EYDw9HhTMnP2b/go-log"
 )
 
@@ -33,11 +34,14 @@ type Datastore struct {
 	path string
 	// length of the dir splay prefix, in bytes of hex digits
 	hexPrefixLen int
+
+	// sychronize all writes and directory changes for added safety
+	sync bool
 }
 
 var _ datastore.Datastore = (*Datastore)(nil)
 
-func New(path string, prefixLen int) (*Datastore, error) {
+func New(path string, prefixLen int, sync bool) (*Datastore, error) {
 	if prefixLen <= 0 || prefixLen > maxPrefixLen {
 		return nil, ErrBadPrefixLen
 	}
@@ -45,6 +49,7 @@ func New(path string, prefixLen int) (*Datastore, error) {
 		path: path,
 		// convert from binary bytes to bytes of hex encoding
 		hexPrefixLen: prefixLen * hex.EncodedLen(1),
+		sync:         sync,
 	}
 	return fs, nil
 }
@@ -80,8 +85,10 @@ func (fs *Datastore) makePrefixDir(dir string) error {
 	// it, the creation of the prefix dir itself might not be
 	// durable yet. Sync the root dir after a successful mkdir of
 	// a prefix dir, just to be paranoid.
-	if err := syncDir(fs.path); err != nil {
-		return err
+	if fs.sync {
+		if err := syncDir(fs.path); err != nil {
+			return err
+		}
 	}
 	return nil
 }
@@ -148,8 +155,10 @@ func (fs *Datastore) doPut(key datastore.Key, val []byte) error {
 	if _, err := tmp.Write(val); err != nil {
 		return err
 	}
-	if err := tmp.Sync(); err != nil {
-		return err
+	if fs.sync {
+		if err := tmp.Sync(); err != nil {
+			return err
+		}
 	}
 	if err := tmp.Close(); err != nil {
 		return err
@@ -162,8 +171,10 @@ func (fs *Datastore) doPut(key datastore.Key, val []byte) error {
 	}
 	removed = true
 
-	if err := syncDir(dir); err != nil {
-		return err
+	if fs.sync {
+		if err := syncDir(dir); err != nil {
+			return err
+		}
 	}
 	return nil
 }
@@ -213,8 +224,10 @@ func (fs *Datastore) putMany(data map[datastore.Key]interface{}) error {
 	// Now we sync everything
 	// sync and close files
 	for fi, _ := range files {
-		if err := fi.Sync(); err != nil {
-			return err
+		if fs.sync {
+			if err := fi.Sync(); err != nil {
+				return err
+			}
 		}
 
 		if err := fi.Close(); err != nil {
@@ -236,15 +249,17 @@ func (fs *Datastore) putMany(data map[datastore.Key]interface{}) error {
 	}
 
 	// now sync the dirs for those files
-	for _, dir := range dirsToSync {
-		if err := syncDir(dir); err != nil {
-			return err
+	if fs.sync {
+		for _, dir := range dirsToSync {
+			if err := syncDir(dir); err != nil {
+				return err
+			}
 		}
-	}
 
-	// sync top flatfs dir
-	if err := syncDir(fs.path); err != nil {
-		return err
+		// sync top flatfs dir
+		if err := syncDir(fs.path); err != nil {
+			return err
+		}
 	}
 
 	return nil

Godeps/_workspace/src/github.com/jbenet/go-datastore/flatfs/flatfs_test.go

@@ -8,11 +8,12 @@ import (
 	"runtime"
 	"testing"
 
-	rand "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/dustin/randbo"
 	"github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
 	"github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/flatfs"
 	"github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/query"
 	dstest "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/test"
+
+	rand "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/dustin/randbo"
 )
 
 func tempdir(t testing.TB) (path string, cleanup func()) {
@@ -34,7 +35,7 @@ func TestBadPrefixLen(t *testing.T) {
 	defer cleanup()
 
 	for i := 0; i > -3; i-- {
-		_, err := flatfs.New(temp, 0)
+		_, err := flatfs.New(temp, i, false)
 		if g, e := err, flatfs.ErrBadPrefixLen; g != e {
 			t.Errorf("expected ErrBadPrefixLen, got: %v", g)
 		}
@@ -45,7 +46,7 @@ func TestPutBadValueType(t *testing.T) {
 	temp, cleanup := tempdir(t)
 	defer cleanup()
 
-	fs, err := flatfs.New(temp, 2)
+	fs, err := flatfs.New(temp, 2, false)
 	if err != nil {
 		t.Fatalf("New fail: %v\n", err)
 	}
@@ -60,7 +61,7 @@ func TestPut(t *testing.T) {
 	temp, cleanup := tempdir(t)
 	defer cleanup()
 
-	fs, err := flatfs.New(temp, 2)
+	fs, err := flatfs.New(temp, 2, false)
 	if err != nil {
 		t.Fatalf("New fail: %v\n", err)
 	}
@@ -75,7 +76,7 @@ func TestGet(t *testing.T) {
 	temp, cleanup := tempdir(t)
 	defer cleanup()
 
-	fs, err := flatfs.New(temp, 2)
+	fs, err := flatfs.New(temp, 2, false)
 	if err != nil {
 		t.Fatalf("New fail: %v\n", err)
 	}
@@ -103,7 +104,7 @@ func TestPutOverwrite(t *testing.T) {
 	temp, cleanup := tempdir(t)
 	defer cleanup()
 
-	fs, err := flatfs.New(temp, 2)
+	fs, err := flatfs.New(temp, 2, false)
 	if err != nil {
 		t.Fatalf("New fail: %v\n", err)
 	}
@@ -135,7 +136,7 @@ func TestGetNotFoundError(t *testing.T) {
 	temp, cleanup := tempdir(t)
 	defer cleanup()
 
-	fs, err := flatfs.New(temp, 2)
+	fs, err := flatfs.New(temp, 2, false)
 	if err != nil {
 		t.Fatalf("New fail: %v\n", err)
 	}
@@ -153,7 +154,7 @@ func TestStorage(t *testing.T) {
 	const prefixLen = 2
 	const prefix = "7175"
 	const target = prefix + string(os.PathSeparator) + "71757578.data"
-	fs, err := flatfs.New(temp, prefixLen)
+	fs, err := flatfs.New(temp, prefixLen, false)
 	if err != nil {
 		t.Fatalf("New fail: %v\n", err)
 	}
@@ -208,7 +209,7 @@ func TestHasNotFound(t *testing.T) {
 	temp, cleanup := tempdir(t)
 	defer cleanup()
 
-	fs, err := flatfs.New(temp, 2)
+	fs, err := flatfs.New(temp, 2, false)
 	if err != nil {
 		t.Fatalf("New fail: %v\n", err)
 	}
@@ -226,7 +227,7 @@ func TestHasFound(t *testing.T) {
 	temp, cleanup := tempdir(t)
 	defer cleanup()
 
-	fs, err := flatfs.New(temp, 2)
+	fs, err := flatfs.New(temp, 2, false)
 	if err != nil {
 		t.Fatalf("New fail: %v\n", err)
 	}
@@ -248,7 +249,7 @@ func TestDeleteNotFound(t *testing.T) {
 	temp, cleanup := tempdir(t)
 	defer cleanup()
 
-	fs, err := flatfs.New(temp, 2)
+	fs, err := flatfs.New(temp, 2, false)
 	if err != nil {
 		t.Fatalf("New fail: %v\n", err)
 	}
@@ -263,7 +264,7 @@ func TestDeleteFound(t *testing.T) {
 	temp, cleanup := tempdir(t)
 	defer cleanup()
 
-	fs, err := flatfs.New(temp, 2)
+	fs, err := flatfs.New(temp, 2, false)
 	if err != nil {
 		t.Fatalf("New fail: %v\n", err)
 	}
@@ -288,7 +289,7 @@ func TestQuerySimple(t *testing.T) {
 	temp, cleanup := tempdir(t)
 	defer cleanup()
 
-	fs, err := flatfs.New(temp, 2)
+	fs, err := flatfs.New(temp, 2, false)
 	if err != nil {
 		t.Fatalf("New fail: %v\n", err)
 	}
@@ -324,7 +325,7 @@ func TestBatchPut(t *testing.T) {
 	temp, cleanup := tempdir(t)
 	defer cleanup()
 
-	fs, err := flatfs.New(temp, 2)
+	fs, err := flatfs.New(temp, 2, false)
 	if err != nil {
 		t.Fatalf("New fail: %v\n", err)
 	}
@@ -336,7 +337,7 @@ func TestBatchDelete(t *testing.T) {
 	temp, cleanup := tempdir(t)
 	defer cleanup()
 
-	fs, err := flatfs.New(temp, 2)
+	fs, err := flatfs.New(temp, 2, false)
 	if err != nil {
 		t.Fatalf("New fail: %v\n", err)
 	}
@@ -359,7 +360,7 @@ func BenchmarkConsecutivePut(b *testing.B) {
 	temp, cleanup := tempdir(b)
 	defer cleanup()
 
-	fs, err := flatfs.New(temp, 2)
+	fs, err := flatfs.New(temp, 2, false)
 	if err != nil {
 		b.Fatalf("New fail: %v\n", err)
 	}
@@ -389,7 +390,7 @@ func BenchmarkBatchedPut(b *testing.B) {
 	temp, cleanup := tempdir(b)
 	defer cleanup()
 
-	fs, err := flatfs.New(temp, 2)
+	fs, err := flatfs.New(temp, 2, false)
 	if err != nil {
 		b.Fatalf("New fail: %v\n", err)
 	}

Godeps/_workspace/src/github.com/jbenet/go-datastore/lru/datastore_test.go

@@ -5,10 +5,11 @@ import (
 	"testing"
 
 	ds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
-	lru "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/lru" // Hook up gocheck into the "go test" runner.
+	lru "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/lru"
 	. "gopkg.in/check.v1"
 )
 
+// Hook up gocheck into the "go test" runner.
 func Test(t *testing.T) { TestingT(t) }
 
 type DSSuite struct{}

Godeps/_workspace/src/github.com/jbenet/go-datastore/timecache/timecache.go

@@ -9,10 +9,10 @@ import (
 	dsq "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/query"
 )
 
+// op keys
 var (
-	putKey = "put"
-	getKey = // op keys
-	"get"
+	putKey    = "put"
+	getKey    = "get"
 	hasKey    = "has"
 	deleteKey = "delete"
 )

repo/config/datastore.go

@@ -16,6 +16,7 @@ type Datastore struct {
 	GCPeriod           string // in ns, us, ms, s, m, h
 
 	Params *json.RawMessage
+	NoSync bool
 }
 
 func (d *Datastore) ParamData() []byte {

repo/fsrepo/defaultds.go

@@ -39,7 +39,8 @@ func openDefaultDatastore(r *FSRepo) (repo.Datastore, error) {
 	// including "/" from datastore.Key and 2 bytes from multihash. To
 	// reach a uniform 256-way split, we need approximately 4 bytes of
 	// prefix.
-	blocksDS, err := flatfs.New(path.Join(r.path, flatfsDirectory), 4)
+	syncfs := !r.config.Datastore.NoSync
+	blocksDS, err := flatfs.New(path.Join(r.path, flatfsDirectory), 4, syncfs)
 	if err != nil {
 		return nil, fmt.Errorf("unable to open flatfs datastore: %v", err)
 	}