Skip to content

G
go-ipfs
提交 c154f7f0 作者: Łukasz Magiera
操作

Move datastores to plugins 

License: MIT
Signed-off-by: 's avatarŁukasz Magiera <magik6k@gmail.com>
上级 67d9bb98
正在显示 包含 310 行增加 和 188 行删除
plugin/loader/preload.go
...@@ -2,7 +2,10 @@ package loader ...@@ -2,7 +2,10 @@ package loader
import ( import (
"github.com/ipfs/go-ipfs/plugin" "github.com/ipfs/go-ipfs/plugin"
pluginbadgerds "github.com/ipfs/go-ipfs/plugin/plugins/badgerds"
pluginflatfs "github.com/ipfs/go-ipfs/plugin/plugins/flatfs"
pluginipldgit "github.com/ipfs/go-ipfs/plugin/plugins/git" pluginipldgit "github.com/ipfs/go-ipfs/plugin/plugins/git"
pluginlevelds "github.com/ipfs/go-ipfs/plugin/plugins/levelds"
) )
// DO NOT EDIT THIS FILE // DO NOT EDIT THIS FILE
...@@ -11,4 +14,7 @@ import ( ...@@ -11,4 +14,7 @@ import (
var preloadPlugins = []plugin.Plugin{ var preloadPlugins = []plugin.Plugin{
pluginipldgit.Plugins[0], pluginipldgit.Plugins[0],
pluginbadgerds.Plugins[0],
pluginflatfs.Plugins[0],
pluginlevelds.Plugins[0],
} }
plugin/loader/preload_list
...@@ -4,3 +4,7 @@ ...@@ -4,3 +4,7 @@
# name go-path number of the sub-plugin # name go-path number of the sub-plugin
ipldgit github.com/ipfs/go-ipfs/plugin/plugins/git 0 ipldgit github.com/ipfs/go-ipfs/plugin/plugins/git 0
badgerds github.com/ipfs/go-ipfs/plugin/plugins/badgerds 0
flatfs github.com/ipfs/go-ipfs/plugin/plugins/flatfs 0
levelds github.com/ipfs/go-ipfs/plugin/plugins/levelds 0
plugin/plugins/Rules.mk
include mk/header.mk include mk/header.mk
$(d)_plugins:=$(d)/git $(d)_plugins:=$(d)/git $(d)/badgerds $(d)/flatfs $(d)/levelds
$(d)_plugins_so:=$(addsuffix .so,$($(d)_plugins)) $(d)_plugins_so:=$(addsuffix .so,$($(d)_plugins))
$(d)_plugins_main:=$(addsuffix /main/main.go,$($(d)_plugins)) $(d)_plugins_main:=$(addsuffix /main/main.go,$($(d)_plugins))
......
plugin/plugins/badgerds/badgerds.go 0 → 100644
package badgerds
import (
"fmt"
"os"
"path/filepath"
"github.com/ipfs/go-ipfs/plugin"
"github.com/ipfs/go-ipfs/repo"
"github.com/ipfs/go-ipfs/repo/fsrepo"
humanize "gx/ipfs/QmPSBJL4momYnE7DcUyk2DVhD6rH488ZmHBGLbxNdhU44K/go-humanize"
badgerds "gx/ipfs/QmaixNkKwtinV3umL5VD1VDD5CQjnZhXY31awM2YHTzbui/go-ds-badger"
)
// Plugins is exported list of plugins that will be loaded
var Plugins = []plugin.Plugin{
&badgerdsPlugin{},
}
type badgerdsPlugin struct{}
var _ plugin.PluginDatastore = (*badgerdsPlugin)(nil)
func (*badgerdsPlugin) Name() string {
return "ds-badgerds"
}
func (*badgerdsPlugin) Version() string {
return "0.1.0"
}
func (*badgerdsPlugin) Init() error {
return nil
}
func (*badgerdsPlugin) DatastoreTypeName() string {
return "badgerds"
}
type datastoreConfig struct {
path string
syncWrites bool
vlogFileSize int64
}
// BadgerdsDatastoreConfig returns a configuration stub for a badger datastore
// from the given parameters
func (*badgerdsPlugin) DatastoreConfigParser() fsrepo.ConfigFromMap {
return func(params map[string]interface{}) (fsrepo.DatastoreConfig, error) {
var c datastoreConfig
var ok bool
c.path, ok = params["path"].(string)
if !ok {
return nil, fmt.Errorf("'path' field is missing or not string")
}
sw, ok := params["syncWrites"]
if !ok {
c.syncWrites = true
} else {
if swb, ok := sw.(bool); ok {
c.syncWrites = swb
} else {
return nil, fmt.Errorf("'syncWrites' field was not a boolean")
}
}
vls, ok := params["vlogFileSize"]
if !ok {
// default to 1GiB
c.vlogFileSize = badgerds.DefaultOptions.ValueLogFileSize
} else {
if vlogSize, ok := vls.(string); ok {
s, err := humanize.ParseBytes(vlogSize)
if err != nil {
return nil, err
}
c.vlogFileSize = int64(s)
} else {
return nil, fmt.Errorf("'vlogFileSize' field was not a string")
}
}
return &c, nil
}
}
func (c *datastoreConfig) DiskSpec() fsrepo.DiskSpec {
return map[string]interface{}{
"type": "badgerds",
"path": c.path,
}
}
func (c *datastoreConfig) Create(path string) (repo.Datastore, error) {
p := c.path
if !filepath.IsAbs(p) {
p = filepath.Join(path, p)
}
err := os.MkdirAll(p, 0755)
if err != nil {
return nil, err
}
defopts := badgerds.DefaultOptions
defopts.SyncWrites = c.syncWrites
defopts.ValueLogFileSize = c.vlogFileSize
return badgerds.NewDatastore(p, &defopts)
}
plugin/plugins/flatfs/flatfs.go 0 → 100644
package flatfs
import (
"fmt"
"path/filepath"
"github.com/ipfs/go-ipfs/plugin"
"github.com/ipfs/go-ipfs/repo"
"github.com/ipfs/go-ipfs/repo/fsrepo"
flatfs "gx/ipfs/QmVFboKxbVJZMJAoFdvX6q4hzvXFkbWCE8DejnqrQV4ZtN/go-ds-flatfs"
)
// Plugins is exported list of plugins that will be loaded
var Plugins = []plugin.Plugin{
&flatfsPlugin{},
}
type flatfsPlugin struct{}
var _ plugin.PluginDatastore = (*flatfsPlugin)(nil)
func (*flatfsPlugin) Name() string {
return "ds-flatfs"
}
func (*flatfsPlugin) Version() string {
return "0.1.0"
}
func (*flatfsPlugin) Init() error {
return nil
}
func (*flatfsPlugin) DatastoreTypeName() string {
return "flatfs"
}
type datastoreConfig struct {
path string
shardFun *flatfs.ShardIdV1
syncField bool
}
// BadgerdsDatastoreConfig returns a configuration stub for a badger datastore
// from the given parameters
func (*flatfsPlugin) DatastoreConfigParser() fsrepo.ConfigFromMap {
return func(params map[string]interface{}) (fsrepo.DatastoreConfig, error) {
var c datastoreConfig
var ok bool
var err error
c.path, ok = params["path"].(string)
if !ok {
return nil, fmt.Errorf("'path' field is missing or not boolean")
}
sshardFun, ok := params["shardFunc"].(string)
if !ok {
return nil, fmt.Errorf("'shardFunc' field is missing or not a string")
}
c.shardFun, err = flatfs.ParseShardFunc(sshardFun)
if err != nil {
return nil, err
}
c.syncField, ok = params["sync"].(bool)
if !ok {
return nil, fmt.Errorf("'sync' field is missing or not boolean")
}
return &c, nil
}
}
func (c *datastoreConfig) DiskSpec() fsrepo.DiskSpec {
return map[string]interface{}{
"type": "flatfs",
"path": c.path,
"shardFunc": c.shardFun.String(),
}
}
func (c *datastoreConfig) Create(path string) (repo.Datastore, error) {
p := c.path
if !filepath.IsAbs(p) {
p = filepath.Join(path, p)
}
return flatfs.CreateOrOpen(p, c.shardFun, c.syncField)
}
plugin/plugins/levelds/levelds.go 0 → 100644
package levelds
import (
"fmt"
"path/filepath"
"github.com/ipfs/go-ipfs/plugin"
"github.com/ipfs/go-ipfs/repo"
"github.com/ipfs/go-ipfs/repo/fsrepo"
ldbopts "gx/ipfs/QmbBhyDKsY4mbY6xsKt3qu9Y7FPvMJ6qbD8AMjYYvPRw1g/goleveldb/leveldb/opt"
levelds "gx/ipfs/QmccqjKZUTqp4ikWNyAbjBuP5HEdqSqRuAr9mcEhYab54a/go-ds-leveldb"
)
// Plugins is exported list of plugins that will be loaded
var Plugins = []plugin.Plugin{
&leveldsPlugin{},
}
type leveldsPlugin struct{}
var _ plugin.PluginDatastore = (*leveldsPlugin)(nil)
func (*leveldsPlugin) Name() string {
return "ds-level"
}
func (*leveldsPlugin) Version() string {
return "0.1.0"
}
func (*leveldsPlugin) Init() error {
return nil
}
func (*leveldsPlugin) DatastoreTypeName() string {
return "levelds"
}
type datastoreConfig struct {
path string
compression ldbopts.Compression
}
// BadgerdsDatastoreConfig returns a configuration stub for a badger datastore
// from the given parameters
func (*leveldsPlugin) DatastoreConfigParser() fsrepo.ConfigFromMap {
return func(params map[string]interface{}) (fsrepo.DatastoreConfig, error) {
var c datastoreConfig
var ok bool
c.path, ok = params["path"].(string)
if !ok {
return nil, fmt.Errorf("'path' field is missing or not string")
}
switch cm := params["compression"].(string); cm {
case "none":
c.compression = ldbopts.NoCompression
case "snappy":
c.compression = ldbopts.SnappyCompression
case "":
c.compression = ldbopts.DefaultCompression
default:
return nil, fmt.Errorf("unrecognized value for compression: %s", cm)
}
return &c, nil
}
}
func (c *datastoreConfig) DiskSpec() fsrepo.DiskSpec {
return map[string]interface{}{
"type": "levelds",
"path": c.path,
}
}
func (c *datastoreConfig) Create(path string) (repo.Datastore, error) {
p := c.path
if !filepath.IsAbs(p) {
p = filepath.Join(path, p)
}
return levelds.NewDatastore(p, &levelds.Options{
Compression: c.compression,
})
}
repo/fsrepo/datastores.go
...@@ -4,20 +4,13 @@ import ( ...@@ -4,20 +4,13 @@ import (
"bytes" "bytes"
"encoding/json" "encoding/json"
"fmt" "fmt"
"os"
"path/filepath"
"sort" "sort"
repo "github.com/ipfs/go-ipfs/repo" "github.com/ipfs/go-ipfs/repo"
humanize "gx/ipfs/QmPSBJL4momYnE7DcUyk2DVhD6rH488ZmHBGLbxNdhU44K/go-humanize" "gx/ipfs/QmQS6UXi1R87y9nEgnCNmG6YfMzvBSLir7xUheMNFP3hoe/go-ds-measure"
measure "gx/ipfs/QmQS6UXi1R87y9nEgnCNmG6YfMzvBSLir7xUheMNFP3hoe/go-ds-measure"
flatfs "gx/ipfs/QmVFboKxbVJZMJAoFdvX6q4hzvXFkbWCE8DejnqrQV4ZtN/go-ds-flatfs"
ds "gx/ipfs/QmaRb5yNXKonhbkpNxNawoydk4N6es6b4fPj19sjEKsh5D/go-datastore" ds "gx/ipfs/QmaRb5yNXKonhbkpNxNawoydk4N6es6b4fPj19sjEKsh5D/go-datastore"
mount "gx/ipfs/QmaRb5yNXKonhbkpNxNawoydk4N6es6b4fPj19sjEKsh5D/go-datastore/mount" "gx/ipfs/QmaRb5yNXKonhbkpNxNawoydk4N6es6b4fPj19sjEKsh5D/go-datastore/mount"
badgerds "gx/ipfs/QmaixNkKwtinV3umL5VD1VDD5CQjnZhXY31awM2YHTzbui/go-ds-badger"
ldbopts "gx/ipfs/QmbBhyDKsY4mbY6xsKt3qu9Y7FPvMJ6qbD8AMjYYvPRw1g/goleveldb/leveldb/opt"
levelds "gx/ipfs/QmccqjKZUTqp4ikWNyAbjBuP5HEdqSqRuAr9mcEhYab54a/go-ds-leveldb"
) )
// ConfigFromMap creates a new datastore config from a map // ConfigFromMap creates a new datastore config from a map
...@@ -63,13 +56,10 @@ var datastores map[string]ConfigFromMap ...@@ -63,13 +56,10 @@ var datastores map[string]ConfigFromMap
func init() { func init() {
datastores = map[string]ConfigFromMap{ datastores = map[string]ConfigFromMap{
"mount": MountDatastoreConfig, "mount": MountDatastoreConfig,
"flatfs": FlatfsDatastoreConfig, "mem": MemDatastoreConfig,
"levelds": LeveldsDatastoreConfig, "log": LogDatastoreConfig,
"badgerds": BadgerdsDatastoreConfig, "measure": MeasureDatastoreConfig,
"mem": MemDatastoreConfig,
"log": LogDatastoreConfig,
"measure": MeasureDatastoreConfig,
} }
} }
...@@ -170,103 +160,6 @@ func (c *mountDatastoreConfig) Create(path string) (repo.Datastore, error) { ...@@ -170,103 +160,6 @@ func (c *mountDatastoreConfig) Create(path string) (repo.Datastore, error) {
return mount.New(mounts), nil return mount.New(mounts), nil
} }
type flatfsDatastoreConfig struct {
path string
shardFun *flatfs.ShardIdV1
syncField bool
}
// FlatfsDatastoreConfig returns a flatfs DatastoreConfig from a spec
func FlatfsDatastoreConfig(params map[string]interface{}) (DatastoreConfig, error) {
var c flatfsDatastoreConfig
var ok bool
var err error
c.path, ok = params["path"].(string)
if !ok {
return nil, fmt.Errorf("'path' field is missing or not boolean")
}
sshardFun, ok := params["shardFunc"].(string)
if !ok {
return nil, fmt.Errorf("'shardFunc' field is missing or not a string")
}
c.shardFun, err = flatfs.ParseShardFunc(sshardFun)
if err != nil {
return nil, err
}
c.syncField, ok = params["sync"].(bool)
if !ok {
return nil, fmt.Errorf("'sync' field is missing or not boolean")
}
return &c, nil
}
func (c *flatfsDatastoreConfig) DiskSpec() DiskSpec {
return map[string]interface{}{
"type": "flatfs",
"path": c.path,
"shardFunc": c.shardFun.String(),
}
}
func (c *flatfsDatastoreConfig) Create(path string) (repo.Datastore, error) {
p := c.path
if !filepath.IsAbs(p) {
p = filepath.Join(path, p)
}
return flatfs.CreateOrOpen(p, c.shardFun, c.syncField)
}
type leveldsDatastoreConfig struct {
path string
compression ldbopts.Compression
}
// LeveldsDatastoreConfig returns a levelds DatastoreConfig from a spec
func LeveldsDatastoreConfig(params map[string]interface{}) (DatastoreConfig, error) {
var c leveldsDatastoreConfig
var ok bool
c.path, ok = params["path"].(string)
if !ok {
return nil, fmt.Errorf("'path' field is missing or not string")
}
switch cm := params["compression"].(string); cm {
case "none":
c.compression = ldbopts.NoCompression
case "snappy":
c.compression = ldbopts.SnappyCompression
case "":
c.compression = ldbopts.DefaultCompression
default:
return nil, fmt.Errorf("unrecognized value for compression: %s", cm)
}
return &c, nil
}
func (c *leveldsDatastoreConfig) DiskSpec() DiskSpec {
return map[string]interface{}{
"type": "levelds",
"path": c.path,
}
}
func (c *leveldsDatastoreConfig) Create(path string) (repo.Datastore, error) {
p := c.path
if !filepath.IsAbs(p) {
p = filepath.Join(path, p)
}
return levelds.NewDatastore(p, &levelds.Options{
Compression: c.compression,
})
}
type memDatastoreConfig struct { type memDatastoreConfig struct {
cfg map[string]interface{} cfg map[string]interface{}
} }
...@@ -352,76 +245,3 @@ func (c measureDatastoreConfig) Create(path string) (repo.Datastore, error) { ...@@ -352,76 +245,3 @@ func (c measureDatastoreConfig) Create(path string) (repo.Datastore, error) {
} }
return measure.New(c.prefix, child), nil return measure.New(c.prefix, child), nil
} }
type badgerdsDatastoreConfig struct {
path string
syncWrites bool
vlogFileSize int64
}
// BadgerdsDatastoreConfig returns a configuration stub for a badger datastore
// from the given parameters
func BadgerdsDatastoreConfig(params map[string]interface{}) (DatastoreConfig, error) {
var c badgerdsDatastoreConfig
var ok bool
c.path, ok = params["path"].(string)
if !ok {
return nil, fmt.Errorf("'path' field is missing or not string")
}
sw, ok := params["syncWrites"]
if !ok {
c.syncWrites = true
} else {
if swb, ok := sw.(bool); ok {
c.syncWrites = swb
} else {
return nil, fmt.Errorf("'syncWrites' field was not a boolean")
}
}
vls, ok := params["vlogFileSize"]
if !ok {
// default to 1GiB
c.vlogFileSize = badgerds.DefaultOptions.ValueLogFileSize
} else {
if vlogSize, ok := vls.(string); ok {
s, err := humanize.ParseBytes(vlogSize)
if err != nil {
return nil, err
}
c.vlogFileSize = int64(s)
} else {
return nil, fmt.Errorf("'vlogFileSize' field was not a string")
}
}
return &c, nil
}
func (c *badgerdsDatastoreConfig) DiskSpec() DiskSpec {
return map[string]interface{}{
"type": "badgerds",
"path": c.path,
}
}
func (c *badgerdsDatastoreConfig) Create(path string) (repo.Datastore, error) {
p := c.path
if !filepath.IsAbs(p) {
p = filepath.Join(path, p)
}
err := os.MkdirAll(p, 0755)
if err != nil {
return nil, err
}
defopts := badgerds.DefaultOptions
defopts.SyncWrites = c.syncWrites
defopts.ValueLogFileSize = c.vlogFileSize
return badgerds.NewDatastore(p, &defopts)
}
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论