implement metadata node for unixfs and other

core/coreunix/metadata.go

+package coreunix
+
+import (
+	dag "github.com/jbenet/go-ipfs/merkledag"
+	ft "github.com/jbenet/go-ipfs/unixfs"
+	u "github.com/jbenet/go-ipfs/util"
+)
+
+func AddMetadataTo(key u.Key, m *ft.Metadata, dserv dag.DAGService) (u.Key, error) {
+	nd, err := dserv.Get(key)
+	if err != nil {
+		return "", err
+	}
+
+	mdnode := new(dag.Node)
+	mdata, err := ft.BytesForMetadata(m)
+	if err != nil {
+		return "", err
+	}
+
+	mdnode.Data = mdata
+	err = mdnode.AddNodeLinkClean("file", nd)
+	if err != nil {
+		return "", err
+	}
+
+	return dserv.Add(mdnode)
+}
+
+func Metadata(key u.Key, dserv dag.DAGService) (*ft.Metadata, error) {
+	nd, err := dserv.Get(key)
+	if err != nil {
+		return nil, err
+	}
+
+	return ft.MetadataFromBytes(nd.Data)
+}

core/coreunix/metadata_test.go

+package coreunix
+
+import (
+	"bytes"
+	"io/ioutil"
+	"testing"
+
+	context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"
+	ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
+	dssync "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/sync"
+	bstore "github.com/jbenet/go-ipfs/blocks/blockstore"
+	bserv "github.com/jbenet/go-ipfs/blockservice"
+	offline "github.com/jbenet/go-ipfs/exchange/offline"
+	importer "github.com/jbenet/go-ipfs/importer"
+	chunk "github.com/jbenet/go-ipfs/importer/chunk"
+	merkledag "github.com/jbenet/go-ipfs/merkledag"
+	ft "github.com/jbenet/go-ipfs/unixfs"
+	uio "github.com/jbenet/go-ipfs/unixfs/io"
+	u "github.com/jbenet/go-ipfs/util"
+)
+
+func getDagservAndPinner(t *testing.T) merkledag.DAGService {
+	db := dssync.MutexWrap(ds.NewMapDatastore())
+	bs := bstore.NewBlockstore(db)
+	blockserv, err := bserv.New(bs, offline.Exchange(bs))
+	if err != nil {
+		t.Fatal(err)
+	}
+	return merkledag.NewDAGService(blockserv)
+}
+
+func TestMetadata(t *testing.T) {
+	// Make some random node
+	ds := getDagservAndPinner(t)
+	data := make([]byte, 1000)
+	u.NewTimeSeededRand().Read(data)
+	r := bytes.NewReader(data)
+	nd, err := importer.BuildDagFromReader(r, ds, nil, chunk.DefaultSplitter)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	k, err := nd.Key()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	m := new(ft.Metadata)
+	m.MimeType = "THIS IS A TEST"
+
+	mdk, err := AddMetadataTo(k, m, ds)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	rec, err := Metadata(mdk, ds)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if rec.MimeType != m.MimeType {
+		t.Fatalf("something went wrong in conversion: '%s' != '%s'", rec.MimeType, m.MimeType)
+	}
+
+	retnode, err := ds.Get(mdk)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	ndr, err := uio.NewDagReader(context.TODO(), retnode, ds)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	out, err := ioutil.ReadAll(ndr)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !bytes.Equal(out, data) {
+		t.Fatal("read incorrect data")
+	}
+}

unixfs/format.go

@@ -126,3 +126,48 @@ func (mb *MultiBlock) FileSize() uint64 {
 func (mb *MultiBlock) NumChildren() int {
 	return len(mb.blocksizes)
 }
+
+type Metadata struct {
+	MimeType string
+	Size     uint64
+}
+
+func MetadataFromBytes(b []byte) (*Metadata, error) {
+	pbd := new(pb.Data)
+	err := proto.Unmarshal(b, pbd)
+	if err != nil {
+		return nil, err
+	}
+	if pbd.GetType() != pb.Data_Metadata {
+		return nil, errors.New("incorrect node type")
+	}
+
+	pbm := new(pb.Metadata)
+	err = proto.Unmarshal(pbd.Data, pbm)
+	if err != nil {
+		return nil, err
+	}
+	md := new(Metadata)
+	md.MimeType = pbm.GetMimeType()
+	return md, nil
+}
+
+func (m *Metadata) Bytes() ([]byte, error) {
+	pbm := new(pb.Metadata)
+	pbm.MimeType = &m.MimeType
+	return proto.Marshal(pbm)
+}
+
+func BytesForMetadata(m *Metadata) ([]byte, error) {
+	pbd := new(pb.Data)
+	pbd.Filesize = proto.Uint64(m.Size)
+	typ := pb.Data_Metadata
+	pbd.Type = &typ
+	mdd, err := m.Bytes()
+	if err != nil {
+		return nil, err
+	}
+
+	pbd.Data = mdd
+	return proto.Marshal(pbd)
+}

unixfs/io/dagreader.go

@@ -70,6 +70,15 @@ func NewDagReader(ctx context.Context, n *mdag.Node, serv mdag.DAGService) (*Dag
 		fallthrough
 	case ftpb.Data_File:
 		return newDataFileReader(ctx, n, pb, serv), nil
+	case ftpb.Data_Metadata:
+		if len(n.Links) == 0 {
+			return nil, errors.New("incorrectly formatted metadata object")
+		}
+		child, err := n.Links[0].GetNode(serv)
+		if err != nil {
+			return nil, err
+		}
+		return NewDagReader(ctx, child, serv)
 	default:
 		return nil, ft.ErrUnrecognizedType
 	}
@@ -118,6 +127,8 @@ func (dr *DagReader) precalcNextBuf() error {
 	case ftpb.Data_Raw:
 		dr.buf = NewRSNCFromBytes(pb.GetData())
 		return nil
+	case ftpb.Data_Metadata:
+		return errors.New("Shouldnt have had metadata object inside file")
 	default:
 		return ft.ErrUnrecognizedType
 	}

unixfs/pb/unixfs.pb.go

@@ -10,10 +10,11 @@ It is generated from these files:
 
 It has these top-level messages:
 	Data
+	Metadata
 */
 package unixfs_pb
 
-import proto "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/gogoprotobuf/proto"
+import proto "code.google.com/p/gogoprotobuf/proto"
 import math "math"
 
 // Reference imports to suppress errors if they are not otherwise used.
@@ -26,17 +27,20 @@ const (
 	Data_Raw       Data_DataType = 0
 	Data_Directory Data_DataType = 1
 	Data_File      Data_DataType = 2
+	Data_Metadata  Data_DataType = 3
 )
 
 var Data_DataType_name = map[int32]string{
 	0: "Raw",
 	1: "Directory",
 	2: "File",
+	3: "Metadata",
 }
 var Data_DataType_value = map[string]int32{
 	"Raw":       0,
 	"Directory": 1,
 	"File":      2,
+	"Metadata":  3,
 }
 
 func (x Data_DataType) Enum() *Data_DataType {
@@ -96,6 +100,22 @@ func (m *Data) GetBlocksizes() []uint64 {
 	return nil
 }
 
+type Metadata struct {
+	MimeType         *string `protobuf:"bytes,1,req" json:"MimeType,omitempty"`
+	XXX_unrecognized []byte  `json:"-"`
+}
+
+func (m *Metadata) Reset()         { *m = Metadata{} }
+func (m *Metadata) String() string { return proto.CompactTextString(m) }
+func (*Metadata) ProtoMessage()    {}
+
+func (m *Metadata) GetMimeType() string {
+	if m != nil && m.MimeType != nil {
+		return *m.MimeType
+	}
+	return ""
+}
+
 func init() {
 	proto.RegisterEnum("unixfs.pb.Data_DataType", Data_DataType_name, Data_DataType_value)
 }

unixfs/pb/unixfs.proto

@@ -5,6 +5,7 @@ message Data {
 		Raw = 0;
 		Directory = 1;
 		File = 2;
+		Metadata = 3;
 	}
 
 	required DataType Type = 1;
@@ -12,3 +13,7 @@ message Data {
 	optional uint64 filesize = 3;
 	repeated uint64 blocksizes = 4;
 }
+
+message Metadata {
+	required string MimeType = 1;
+}

util/key.go

@@ -23,6 +23,10 @@ func (k Key) Pretty() string {
 	return k.B58String()
 }
 
+func (k Key) ToMultihash() mh.Multihash {
+	return mh.Multihash(k)
+}
+
 // B58String returns Key in a b58 encoded string
 func (k Key) B58String() string {
 	return B58KeyEncode(k)