flesh out pinning object, needs tests and cli wiring still

Godeps/Godeps.json

 {
 	"ImportPath": "github.com/jbenet/go-ipfs",
-	"GoVersion": "go1.3",
+	"GoVersion": "go1.3.3",
 	"Packages": [
 		"./..."
 	],

Godeps/_workspace/src/github.com/jbenet/datastore.go/keytransform/doc.go

+// Package keytransform introduces a Datastore Shim that transforms keys before
+// passing them to its child. It can be used to manipulate what keys look like
+// to the user, for example namespacing keys, reversing them, etc.
+//
+// Use the Wrap function to wrap a datastore with any KeyTransform.
+// A KeyTransform is simply an interface with two functions, a conversion and
+// its inverse. For example:
+//
+//   import (
+//     ktds "github.com/jbenet/datastore.go/keytransform"
+//     ds "github.com/jbenet/datastore.go"
+//   )
+//
+//   func reverseKey(k ds.Key) ds.Key {
+//     return k.Reverse()
+//   }
+//
+//   func invertKeys(d ds.Datastore) {
+//     return ktds.Wrap(d, &ktds.Pair{
+//       Convert: reverseKey,
+//       Invert: reverseKey,  // reverse is its own inverse.
+//     })
+//   }
+//
+package keytransform

Godeps/_workspace/src/github.com/jbenet/datastore.go/keytransform/interface.go

+package keytransform
+
+import ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go"
+
+// KeyMapping is a function that maps one key to annother
+type KeyMapping func(ds.Key) ds.Key
+
+// KeyTransform is an object with a pair of functions for (invertibly)
+// transforming keys
+type KeyTransform interface {
+	ConvertKey(ds.Key) ds.Key
+	InvertKey(ds.Key) ds.Key
+}
+
+// Datastore is a keytransform.Datastore
+type Datastore interface {
+	ds.Shim
+	KeyTransform
+}
+
+// Wrap wraps a given datastore with a KeyTransform function.
+// The resulting wrapped datastore will use the transform on all Datastore
+// operations.
+func Wrap(child ds.Datastore, t KeyTransform) Datastore {
+	if t == nil {
+		panic("t (KeyTransform) is nil")
+	}
+
+	if child == nil {
+		panic("child (ds.Datastore) is nil")
+	}
+
+	return &ktds{child: child, KeyTransform: t}
+}

Godeps/_workspace/src/github.com/jbenet/datastore.go/namespace/doc.go

+// Package namespace introduces a namespace Datastore Shim, which basically
+// mounts the entire child datastore under a prefix.
+//
+// Use the Wrap function to wrap a datastore with any Key prefix. For example:
+//
+//  import (
+//    "fmt"
+//
+//    ds "github.com/jbenet/datastore.go"
+//    nsds "github.com/jbenet/datastore.go/namespace"
+//  )
+//
+//  func main() {
+//    mp := ds.NewMapDatastore()
+//    ns := nsds.Wrap(mp, ds.NewKey("/foo/bar"))
+//
+//    // in the Namespace Datastore:
+//    ns.Put(ds.NewKey("/beep"), "boop")
+//    v2, _ := ns.Get(ds.NewKey("/beep")) // v2 == "boop"
+//
+//    // and, in the underlying MapDatastore:
+//    v3, _ := mp.Get(ds.NewKey("/foo/bar/beep")) // v3 == "boop"
+//  }
+package namespace

Godeps/_workspace/src/github.com/jbenet/datastore.go/namespace/example_test.go

+package namespace_test
+
+import (
+	"fmt"
+
+	ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go"
+	nsds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go/namespace"
+)
+
+func Example() {
+	mp := ds.NewMapDatastore()
+	ns := nsds.Wrap(mp, ds.NewKey("/foo/bar"))
+
+	k := ds.NewKey("/beep")
+	v := "boop"
+
+	ns.Put(k, v)
+	fmt.Printf("ns.Put %s %s\n", k, v)
+
+	v2, _ := ns.Get(k)
+	fmt.Printf("ns.Get %s -> %s\n", k, v2)
+
+	k3 := ds.NewKey("/foo/bar/beep")
+	v3, _ := mp.Get(k3)
+	fmt.Printf("mp.Get %s -> %s\n", k3, v3)
+	// Output:
+	// ns.Put /beep -> boop
+	// ns.Get /beep -> boop
+	// mp.Get /foo/bar/beep -> boop
+}

Godeps/_workspace/src/github.com/jbenet/datastore.go/namespace/namespace.go

+package namespace
+
+import (
+	"fmt"
+	"strings"
+
+	ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go"
+	ktds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go/keytransform"
+)
+
+// PrefixTransform constructs a KeyTransform with a pair of functions that
+// add or remove the given prefix key.
+//
+// Warning: will panic if prefix not found when it should be there. This is
+// to avoid insidious data inconsistency errors.
+func PrefixTransform(prefix ds.Key) ktds.KeyTransform {
+	return &ktds.Pair{
+
+		// Convert adds the prefix
+		Convert: func(k ds.Key) ds.Key {
+			return prefix.Child(k)
+		},
+
+		// Invert removes the prefix. panics if prefix not found.
+		Invert: func(k ds.Key) ds.Key {
+			if !prefix.IsAncestorOf(k) {
+				fmt.Errorf("Expected prefix (%s) in key (%s)", prefix, k)
+				panic("expected prefix not found")
+			}
+
+			s := strings.TrimPrefix(k.String(), prefix.String())
+			return ds.NewKey(s)
+		},
+	}
+}
+
+// Wrap wraps a given datastore with a key-prefix.
+func Wrap(child ds.Datastore, prefix ds.Key) ktds.Datastore {
+	if child == nil {
+		panic("child (ds.Datastore) is nil")
+	}
+
+	return ktds.Wrap(child, PrefixTransform(prefix))
+}

Godeps/_workspace/src/github.com/jbenet/datastore.go/namespace/namespace_test.go

+package namespace_test
+
+import (
+	"bytes"
+	"sort"
+	"testing"
+
+	ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go"
+	ns "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go/namespace"
+	. "launchpad.net/gocheck"
+)
+
+// Hook up gocheck into the "go test" runner.
+func Test(t *testing.T) { TestingT(t) }
+
+type DSSuite struct{}
+
+var _ = Suite(&DSSuite{})
+
+func (ks *DSSuite) TestBasic(c *C) {
+
+	mpds := ds.NewMapDatastore()
+	nsds := ns.Wrap(mpds, ds.NewKey("abc"))
+
+	keys := strsToKeys([]string{
+		"foo",
+		"foo/bar",
+		"foo/bar/baz",
+		"foo/barb",
+		"foo/bar/bazb",
+		"foo/bar/baz/barb",
+	})
+
+	for _, k := range keys {
+		err := nsds.Put(k, []byte(k.String()))
+		c.Check(err, Equals, nil)
+	}
+
+	for _, k := range keys {
+		v1, err := nsds.Get(k)
+		c.Check(err, Equals, nil)
+		c.Check(bytes.Equal(v1.([]byte), []byte(k.String())), Equals, true)
+
+		v2, err := mpds.Get(ds.NewKey("abc").Child(k))
+		c.Check(err, Equals, nil)
+		c.Check(bytes.Equal(v2.([]byte), []byte(k.String())), Equals, true)
+	}
+
+	listA, errA := mpds.KeyList()
+	listB, errB := nsds.KeyList()
+	c.Check(errA, Equals, nil)
+	c.Check(errB, Equals, nil)
+	c.Check(len(listA), Equals, len(listB))
+
+	// sort them cause yeah.
+	sort.Sort(ds.KeySlice(listA))
+	sort.Sort(ds.KeySlice(listB))
+
+	for i, kA := range listA {
+		kB := listB[i]
+		c.Check(nsds.InvertKey(kA), Equals, kB)
+		c.Check(kA, Equals, nsds.ConvertKey(kB))
+	}
+}
+
+func strsToKeys(strs []string) []ds.Key {
+	keys := make([]ds.Key, len(strs))
+	for i, s := range strs {
+		keys[i] = ds.NewKey(s)
+	}
+	return keys
+}

Godeps/_workspace/src/github.com/jbenet/go-datastore/fs/fs_test.go

@@ -2,6 +2,7 @@ package fs_test
 
 import (
 	"bytes"
+	"sort"
 	"testing"
 
 	ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
@@ -12,10 +13,7 @@ import (
 // Hook up gocheck into the "go test" runner.
 func Test(t *testing.T) { TestingT(t) }
 
-type DSSuite struct {
-	dir string
-	ds  ds.Datastore
-}
+type DSSuite struct{}
 
 var _ = Suite(&DSSuite{})
 
@@ -54,6 +52,22 @@ func (ks *DSSuite) TestBasic(c *C) {
 		c.Check(err, Equals, nil)
 		c.Check(bytes.Equal(v.([]byte), []byte(k.String())), Equals, true)
 	}
+
+	listA, errA := mpds.KeyList()
+	listB, errB := ktds.KeyList()
+	c.Check(errA, Equals, nil)
+	c.Check(errB, Equals, nil)
+	c.Check(len(listA), Equals, len(listB))
+
+	// sort them cause yeah.
+	sort.Sort(ds.KeySlice(listA))
+	sort.Sort(ds.KeySlice(listB))
+
+	for i, kA := range listA {
+		kB := listB[i]
+		c.Check(pair.Invert(kA), Equals, kB)
+		c.Check(kA, Equals, pair.Convert(kB))
+	}
 }
 
 func strsToKeys(strs []string) []ds.Key {

Godeps/_workspace/src/github.com/jbenet/go-datastore/io/io.go

-package leveldb
-
-import (
-	"bytes"
-	"io"
-
-	ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
-)
-
-// CastAsReader does type assertions to find the type of a value and attempts
-// to turn it into an io.Reader. If not possible, will return ds.ErrInvalidType
-func CastAsReader(value interface{}) (io.Reader, error) {
-	switch v := value.(type) {
-	case io.Reader:
-		return v, nil
-
-	case []byte:
-		return bytes.NewReader(v), nil
-
-	case string:
-		return bytes.NewReader([]byte(v)), nil
-
-	default:
-		return nil, ds.ErrInvalidType
-	}
-}
-
-// // CastAsWriter does type assertions to find the type of a value and attempts
-// // to turn it into an io.Writer. If not possible, will return ds.ErrInvalidType
-// func CastAsWriter(value interface{}) (err error) {
-// 	switch v := value.(type) {
-// 	case io.Reader:
-// 		return v, nil
-//
-// 	case []byte:
-// 		return bytes.NewReader(v), nil
-//
-// 	case string:
-// 		return bytes.NewReader([]byte(v)), nil
-//
-// 	default:
-// 		return nil, ds.ErrInvalidType
-// 	}
-// }

blocks/set/dbset.go

@@ -9,19 +9,17 @@ import (
 type datastoreBlockSet struct {
 	dstore ds.Datastore
 	bset   BlockSet
-	prefix string
 }
 
-func NewDBWrapperSet(d ds.Datastore, prefix string, bset BlockSet) BlockSet {
+func NewDBWrapperSet(d ds.Datastore, bset BlockSet) BlockSet {
 	return &datastoreBlockSet{
 		dstore: d,
 		bset:   bset,
-		prefix: prefix,
 	}
 }
 
 func (d *datastoreBlockSet) AddBlock(k util.Key) {
-	err := d.dstore.Put(d.prefixKey(k), []byte{})
+	err := d.dstore.Put(k.DsKey(), []byte{})
 	if err != nil {
 		log.Error("blockset put error: %s", err)
 	}
@@ -32,7 +30,7 @@ func (d *datastoreBlockSet) AddBlock(k util.Key) {
 func (d *datastoreBlockSet) RemoveBlock(k util.Key) {
 	d.bset.RemoveBlock(k)
 	if !d.bset.HasKey(k) {
-		d.dstore.Delete(d.prefixKey(k))
+		d.dstore.Delete(k.DsKey())
 	}
 }
 
@@ -44,6 +42,6 @@ func (d *datastoreBlockSet) GetBloomFilter() bloom.Filter {
 	return d.bset.GetBloomFilter()
 }
 
-func (d *datastoreBlockSet) prefixKey(k util.Key) ds.Key {
-	return (util.Key(d.prefix) + k).DsKey()
+func (d *datastoreBlockSet) GetKeys() []util.Key {
+	return d.bset.GetKeys()
 }

blocks/set/set.go

 package set
 
 import (
+	"errors"
+
+	ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go"
+
 	"github.com/jbenet/go-ipfs/blocks/bloom"
 	"github.com/jbenet/go-ipfs/util"
 )
@@ -12,6 +16,29 @@ type BlockSet interface {
 	RemoveBlock(util.Key)
 	HasKey(util.Key) bool
 	GetBloomFilter() bloom.Filter
+
+	GetKeys() []util.Key
+}
+
+func SimpleSetFromKeys(keys []util.Key) BlockSet {
+	sbs := &simpleBlockSet{blocks: make(map[util.Key]struct{})}
+	for _, k := range keys {
+		sbs.blocks[k] = struct{}{}
+	}
+	return sbs
+}
+
+func SetFromDatastore(d ds.Datastore, k ds.Key) (BlockSet, error) {
+	ikeys, err := d.Get(k)
+	if err != nil {
+		return nil, err
+	}
+
+	keys, ok := ikeys.([]util.Key)
+	if !ok {
+		return nil, errors.New("Incorrect type for keys from datastore")
+	}
+	return SimpleSetFromKeys(keys), nil
 }
 
 func NewSimpleBlockSet() BlockSet {
@@ -42,3 +69,11 @@ func (b *simpleBlockSet) GetBloomFilter() bloom.Filter {
 	}
 	return f
 }
+
+func (b *simpleBlockSet) GetKeys() []util.Key {
+	var out []util.Key
+	for k, _ := range b.blocks {
+		out = append(out, k)
+	}
+	return out
+}

pin/indirect.go

 package pin
 
 import (
+	"errors"
+
 	ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go"
-	bc "github.com/jbenet/go-ipfs/blocks/set"
+	"github.com/jbenet/go-ipfs/blocks/set"
 	"github.com/jbenet/go-ipfs/util"
 )
 
 type indirectPin struct {
-	blockset  bc.BlockSet
+	blockset  set.BlockSet
 	refCounts map[util.Key]int
 }
 
-func loadBlockSet(d ds.Datastore) (bc.BlockSet, map[util.Key]int) {
-	panic("Not yet implemented!")
-	return nil, nil
+func NewIndirectPin(dstore ds.Datastore) *indirectPin {
+	return &indirectPin{
+		blockset:  set.NewDBWrapperSet(dstore, set.NewSimpleBlockSet()),
+		refCounts: make(map[util.Key]int),
+	}
 }
 
-func newIndirectPin(d ds.Datastore) indirectPin {
-	// suppose the blockset actually takes blocks, not just keys
-	bs, rc := loadBlockSet(d)
-	return indirectPin{bs, rc}
+func loadIndirPin(d ds.Datastore, k ds.Key) (*indirectPin, error) {
+	irefcnt, err := d.Get(k)
+	if err != nil {
+		return nil, err
+	}
+	refcnt, ok := irefcnt.(map[util.Key]int)
+	if !ok {
+		return nil, errors.New("invalid type from datastore")
+	}
+
+	var keys []util.Key
+	for k, _ := range refcnt {
+		keys = append(keys, k)
+	}
+
+	return &indirectPin{blockset: set.SimpleSetFromKeys(keys), refCounts: refcnt}, nil
 }
 
 func (i *indirectPin) Increment(k util.Key) {

pin/pin.go

 package pin
 
 import (
+
+	//ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go"
 	ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go"
+	nsds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go/namespace"
 	"github.com/jbenet/go-ipfs/blocks/set"
 	mdag "github.com/jbenet/go-ipfs/merkledag"
 	"github.com/jbenet/go-ipfs/util"
 )
 
+var recursePinDatastoreKey = ds.NewKey("/local/pins/recursive/keys")
+var directPinDatastoreKey = ds.NewKey("/local/pins/direct/keys")
+var indirectPinDatastoreKey = ds.NewKey("/local/pins/indirect/keys")
+
 type Pinner interface {
 	Pin(*mdag.Node, bool) error
 	Unpin(util.Key, bool) error
+	Flush() error
 }
 
 type pinner struct {
 	recursePin set.BlockSet
 	directPin  set.BlockSet
-	indirPin   indirectPin
+	indirPin   *indirectPin
 	dserv      *mdag.DAGService
 	dstore     ds.Datastore
 }
@@ -23,14 +31,17 @@ type pinner struct {
 func NewPinner(dstore ds.Datastore, serv *mdag.DAGService) Pinner {
 
 	// Load set from given datastore...
-	rcset := set.NewDBWrapperSet(dstore, "/pinned/recurse/", set.NewSimpleBlockSet())
-	dirset := set.NewDBWrapperSet(dstore, "/pinned/direct/", set.NewSimpleBlockSet())
+	rcds := nsds.Wrap(dstore, recursePinDatastoreKey)
+	rcset := set.NewDBWrapperSet(rcds, set.NewSimpleBlockSet())
 
-	nsdstore := dstore // WRAP IN NAMESPACE
+	dirds := nsds.Wrap(dstore, directPinDatastoreKey)
+	dirset := set.NewDBWrapperSet(dirds, set.NewSimpleBlockSet())
+
+	nsdstore := nsds.Wrap(dstore, indirectPinDatastoreKey)
 	return &pinner{
 		recursePin: rcset,
 		directPin:  dirset,
-		indirPin:   newIndirectPin(nsdstore),
+		indirPin:   NewIndirectPin(nsdstore),
 		dserv:      serv,
 		dstore:     dstore,
 	}
@@ -126,3 +137,44 @@ func (p *pinner) IsPinned(key util.Key) bool {
 		p.directPin.HasKey(key) ||
 		p.indirPin.HasKey(key)
 }
+
+func LoadPinner(d ds.Datastore) (Pinner, error) {
+	p := new(pinner)
+
+	var err error
+	p.recursePin, err = set.SetFromDatastore(d, recursePinDatastoreKey)
+	if err != nil {
+		return nil, err
+	}
+	p.directPin, err = set.SetFromDatastore(d, directPinDatastoreKey)
+	if err != nil {
+		return nil, err
+	}
+
+	p.indirPin, err = loadIndirPin(d, indirectPinDatastoreKey)
+	if err != nil {
+		return nil, err
+	}
+
+	return p, nil
+}
+
+func (p *pinner) Flush() error {
+	recurse := p.recursePin.GetKeys()
+	err := p.dstore.Put(recursePinDatastoreKey, recurse)
+	if err != nil {
+		return err
+	}
+
+	direct := p.directPin.GetKeys()
+	err = p.dstore.Put(directPinDatastoreKey, direct)
+	if err != nil {
+		return err
+	}
+
+	err = p.dstore.Put(indirectPinDatastoreKey, p.indirPin.refCounts)
+	if err != nil {
+		return err
+	}
+	return nil
+}