S3 datastore support

To test it, set up an S3 bucket (in an AWS region that is not US
Standard, for read-after-write consistency), run `ipfs init`, then
edit `~/.ipfs/config` to say

      "Datastore": {
        "Type": "s3",
        "Region": "us-west-1",
        "Bucket": "mahbukkit",
        "ACL": "private"
      },

with the right values. Set `AWS_ACCESS_KEY_ID` and
`AWS_SECRET_ACCESS_KEY` in the environment and you should be able to
run `ipfs add` and `ipfs cat` and see the bucket be populated.

No automated tests exist, unfortunately. S3 is thorny to simulate.

License: MIT
Signed-off-by: Tommi Virtanen <tv@eagain.net>

repo/config/datastore.go

 package config
 
+import (
+	"encoding/json"
+)
+
 // DefaultDataStoreDirectory is the directory to store all the local IPFS data.
 const DefaultDataStoreDirectory = "datastore"
 
@@ -10,6 +14,22 @@ type Datastore struct {
 	StorageMax         string // in B, kB, kiB, MB, ...
 	StorageGCWatermark int64  // in percentage to multiply on StorageMax
 	GCPeriod           string // in ns, us, ms, s, m, h
+
+	Params *json.RawMessage
+}
+
+func (d *Datastore) ParamData() []byte {
+	if d.Params == nil {
+		return nil
+	}
+
+	return []byte(*d.Params)
+}
+
+type S3Datastore struct {
+	Region string `json:"region"`
+	Bucket string `json:"bucket"`
+	ACL    string `json:"acl"`
 }
 
 // DataStorePath returns the default data store path given a configuration root

repo/fsrepo/datastores.go

+package fsrepo
+
+import (
+	"fmt"
+
+	"github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/crowdmob/goamz/aws"
+	"github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/crowdmob/goamz/s3"
+
+	repo "github.com/ipfs/go-ipfs/repo"
+	config "github.com/ipfs/go-ipfs/repo/config"
+	"github.com/ipfs/go-ipfs/thirdparty/s3-datastore"
+)
+
+func openS3Datastore(params config.S3Datastore) (repo.Datastore, error) {
+	// TODO support credentials files
+	auth, err := aws.EnvAuth()
+	if err != nil {
+		return nil, err
+	}
+
+	region := aws.GetRegion(params.Region)
+	if region.Name == "" {
+		return nil, fmt.Errorf("unknown AWS region: %q", params.Region)
+	}
+
+	if params.Bucket == "" {
+		return nil, fmt.Errorf("invalid S3 bucket: %q", params.Bucket)
+	}
+
+	client := s3.New(auth, region)
+	// There are too many gophermucking s3datastores in my
+	// gophermucking source.
+	return &s3datastore.S3Datastore{
+		Client: client,
+		Bucket: params.Bucket,
+		ACL:    s3.ACL(params.ACL),
+	}, nil
+}

repo/fsrepo/fsrepo.go

 package fsrepo
 
 import (
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
@@ -331,6 +332,18 @@ func (r *FSRepo) openDatastore() error {
 			return err
 		}
 		r.ds = d
+	case "s3":
+		var dscfg config.S3Datastore
+		if err := json.Unmarshal(r.config.Datastore.ParamData(), &dscfg); err != nil {
+			return fmt.Errorf("datastore s3: %v", err)
+		}
+
+		ds, err := openS3Datastore(dscfg)
+		if err != nil {
+			return err
+		}
+
+		r.ds = ds
 	default:
 		return fmt.Errorf("unknown datastore type: %s", r.config.Datastore.Type)
 	}

repo/fsrepo/serialize/serialize_test.go

@@ -14,21 +14,20 @@ func TestConfig(t *testing.T) {
 
 	err := WriteConfigFile(filename, cfgWritten)
 	if err != nil {
-		t.Error(err)
+		t.Fatal(err)
 	}
 	cfgRead, err := Load(filename)
 	if err != nil {
-		t.Error(err)
-		return
+		t.Fatal(err)
 	}
 	if cfgWritten.Identity.PeerID != cfgRead.Identity.PeerID {
-		t.Fail()
+		t.Fatal()
 	}
 	st, err := os.Stat(filename)
 	if err != nil {
 		t.Fatalf("cannot stat config file: %v", err)
 	}
 	if g := st.Mode().Perm(); g&0117 != 0 {
-		t.Errorf("config file should not be executable or accessible to world: %v", g)
+		t.Fatalf("config file should not be executable or accessible to world: %v", g)
 	}
 }

thirdparty/s3-datastore/datastore.go

@@ -67,4 +67,8 @@ func (ds *S3Datastore) Query(q query.Query) (query.Results, error) {
 	return nil, errors.New("TODO implement query for s3 datastore?")
 }
 
+func (ds *S3Datastore) Close() error {
+	return nil
+}
+
 func (ds *S3Datastore) IsThreadSafe() {}