From a852edc763cc038f90792cb68283b56a32d0f3f1 Mon Sep 17 00:00:00 2001 From: "A.Unger" Date: Tue, 6 Oct 2020 16:57:19 +0200 Subject: [PATCH] wip config --- accounts/pkg/config/config.go | 3 +- accounts/pkg/indexer/index/cs3/unique.go | 35 ++++++++ accounts/pkg/indexer/index/disk/non_unique.go | 25 ++++++ accounts/pkg/indexer/index/disk/unique.go | 28 +++++- accounts/pkg/indexer/indexer.go | 69 +++++++++++---- accounts/pkg/indexer/indexer_test.go | 50 ++++++++--- accounts/pkg/indexer/option/option.go | 85 +++++++++++++++++++ accounts/pkg/indexer/registry/registry.go | 14 +++ 8 files changed, 280 insertions(+), 29 deletions(-) create mode 100644 accounts/pkg/indexer/option/option.go create mode 100644 accounts/pkg/indexer/registry/registry.go diff --git a/accounts/pkg/config/config.go b/accounts/pkg/config/config.go index 6240b06b1b..bb22ecf6a4 100644 --- a/accounts/pkg/config/config.go +++ b/accounts/pkg/config/config.go @@ -77,11 +77,12 @@ type CS3 struct { ProviderAddr string DataURL string DataPrefix string + JWTSecret string } // ServiceUser defines the user required for EOS type ServiceUser struct { - UUID string + UUID string Username string UID int64 GID int64 diff --git a/accounts/pkg/indexer/index/cs3/unique.go b/accounts/pkg/indexer/index/cs3/unique.go index 43f8f84f2b..a804d736cf 100644 --- a/accounts/pkg/indexer/index/cs3/unique.go +++ b/accounts/pkg/indexer/index/cs3/unique.go @@ -10,6 +10,9 @@ import ( "github.com/cs3org/reva/pkg/token" "github.com/cs3org/reva/pkg/token/manager/jwt" idxerrs "github.com/owncloud/ocis/accounts/pkg/indexer/errors" + "github.com/owncloud/ocis/accounts/pkg/indexer/index" + "github.com/owncloud/ocis/accounts/pkg/indexer/option" + "github.com/owncloud/ocis/accounts/pkg/indexer/registry" "google.golang.org/grpc/metadata" "io/ioutil" "net/http" @@ -42,6 +45,38 @@ type Config struct { ServiceUserUUID string } +func init() { + registry.IndexConstructorRegistry["cs3"]["unique"] = NewUniqueIndexWithOptions +} + +// NewUniqueIndexWithOptions instantiates a new UniqueIndex instance. Init() should be +// called afterward to ensure correct on-disk structure. +func NewUniqueIndexWithOptions(o ...option.Option) index.Index { + opts := &option.Options{} + for _, opt := range o { + opt(opts) + } + + u := &Unique{ + indexBy: opts.IndexBy, + typeName: opts.TypeName, + filesDir: opts.FilesDir, + indexBaseDir: path.Join(opts.DataDir, "index.cs3"), + indexRootDir: path.Join(path.Join(opts.DataDir, "index.cs3"), strings.Join([]string{"unique", opts.TypeName, opts.IndexBy}, ".")), + cs3conf: &Config{ + JWTSecret: opts.JWTSecret, + }, + dataProvider: dataProviderClient{ + baseURL: singleJoiningSlash(opts.DataURL, opts.DataPrefix), + client: http.Client{ + Transport: http.DefaultTransport, + }, + }, + } + + return u +} + // NewUniqueIndex instantiates a new UniqueIndex instance. Init() should be // called afterward to ensure correct on-disk structure. func NewUniqueIndex(typeName, indexBy, filesDir, indexBaseDir string, cfg *Config) Unique { diff --git a/accounts/pkg/indexer/index/disk/non_unique.go b/accounts/pkg/indexer/index/disk/non_unique.go index aa86a2bcb4..35ceaaee52 100644 --- a/accounts/pkg/indexer/index/disk/non_unique.go +++ b/accounts/pkg/indexer/index/disk/non_unique.go @@ -4,10 +4,14 @@ import ( "errors" "fmt" idxerrs "github.com/owncloud/ocis/accounts/pkg/indexer/errors" + "github.com/owncloud/ocis/accounts/pkg/indexer/index" + "github.com/owncloud/ocis/accounts/pkg/indexer/option" + "github.com/owncloud/ocis/accounts/pkg/indexer/registry" "io/ioutil" "os" "path" "path/filepath" + "strings" ) // NonUniqueIndex is able to index an document by a key which might contain non-unique values @@ -28,6 +32,27 @@ type NonUniqueIndex struct { indexRootDir string } +func init() { + registry.IndexConstructorRegistry["disk"]["non_unique"] = NewNonUniqueIndexWithOptions +} + +// NewNonUniqueIndexWithOptions instantiates a new UniqueIndex instance. Init() should be +// called afterward to ensure correct on-disk structure. +func NewNonUniqueIndexWithOptions(o ...option.Option) index.Index { + opts := &option.Options{} + for _, opt := range o { + opt(opts) + } + + return NonUniqueIndex{ + indexBy: opts.IndexBy, + typeName: opts.TypeName, + filesDir: opts.FilesDir, + indexBaseDir: opts.IndexBaseDir, + indexRootDir: path.Join(opts.IndexBaseDir, strings.Join([]string{"unique", opts.TypeName, opts.IndexBy}, ".")), + } +} + // NewNonUniqueIndex instantiates a new NonUniqueIndex instance. Init() should be // called afterward to ensure correct on-disk structure. func NewNonUniqueIndex(typeName, indexBy, filesDir, indexBaseDir string) NonUniqueIndex { diff --git a/accounts/pkg/indexer/index/disk/unique.go b/accounts/pkg/indexer/index/disk/unique.go index 5556de93c8..7435c97a0e 100644 --- a/accounts/pkg/indexer/index/disk/unique.go +++ b/accounts/pkg/indexer/index/disk/unique.go @@ -4,6 +4,9 @@ import ( "errors" "fmt" idxerrs "github.com/owncloud/ocis/accounts/pkg/indexer/errors" + "github.com/owncloud/ocis/accounts/pkg/indexer/index" + "github.com/owncloud/ocis/accounts/pkg/indexer/option" + "github.com/owncloud/ocis/accounts/pkg/indexer/registry" "os" "path" "path/filepath" @@ -38,6 +41,29 @@ type Unique struct { indexRootDir string } +func init() { + registry.IndexConstructorRegistry["disk"]["unique"] = NewUniqueIndexWithOptions +} + +// NewUniqueIndexWithOptions instantiates a new UniqueIndex instance. Init() should be +// called afterward to ensure correct on-disk structure. +func NewUniqueIndexWithOptions(o ...option.Option) index.Index { + opts := &option.Options{} + for _, opt := range o { + opt(opts) + } + + u := &Unique{ + indexBy: opts.IndexBy, + typeName: opts.TypeName, + filesDir: opts.FilesDir, + indexBaseDir: path.Join(opts.DataDir, "index.disk"), + indexRootDir: path.Join(path.Join(opts.DataDir, "index.disk"), strings.Join([]string{"unique", opts.TypeName, opts.IndexBy}, ".")), + } + + return u +} + // NewUniqueIndex instantiates a new UniqueIndex instance. Init() should be // called afterward to ensure correct on-disk structure. func NewUniqueIndex(typeName, indexBy, filesDir, indexBaseDir string) Unique { @@ -50,7 +76,7 @@ func NewUniqueIndex(typeName, indexBy, filesDir, indexBaseDir string) Unique { } } -func (idx Unique) Init() error { +func (idx *Unique) Init() error { if _, err := os.Stat(idx.filesDir); err != nil { return err } diff --git a/accounts/pkg/indexer/indexer.go b/accounts/pkg/indexer/indexer.go index 2195bdd8f5..7110f8ec76 100644 --- a/accounts/pkg/indexer/indexer.go +++ b/accounts/pkg/indexer/indexer.go @@ -3,16 +3,20 @@ package indexer import ( "fmt" + "github.com/owncloud/ocis/accounts/pkg/config" "github.com/owncloud/ocis/accounts/pkg/indexer/errors" - "github.com/owncloud/ocis/accounts/pkg/indexer/index/disk" + "github.com/owncloud/ocis/accounts/pkg/indexer/index" + "github.com/owncloud/ocis/accounts/pkg/indexer/option" + "github.com/owncloud/ocis/accounts/pkg/indexer/registry" "github.com/rs/zerolog" "path" ) // Indexer is a facade to configure and query over multiple indices. type Indexer struct { - config *Config - indices typeMap + newConfig *config.Config + config *Config + indices typeMap } type Config struct { @@ -21,32 +25,67 @@ type Config struct { Log zerolog.Logger } -func NewIndex(cfg *Config) *Indexer { +func NewIndexer(cfg *Config) *Indexer { return &Indexer{ config: cfg, indices: typeMap{}, } } +func CreateIndexer(cfg *config.Config) *Indexer { + return &Indexer{ + newConfig: cfg, + indices: typeMap{}, + } +} + +func getRegistryStrategy(cfg *config.Config) string { + if cfg.Repo.Disk.Path != "" { + return "disk" + } + + return "cs3" +} + func (i Indexer) AddUniqueIndex(t interface{}, indexBy, pkName, entityDirName string) error { - typeName := getTypeFQN(t) - fullDataPath := path.Join(i.config.DataDir, entityDirName) - indexPath := path.Join(i.config.DataDir, i.config.IndexRootDirName) + strategy := getRegistryStrategy(i.newConfig) + f := registry.IndexConstructorRegistry[strategy]["unique"] + var idx index.Index - idx := disk.NewUniqueIndex(typeName, indexBy, fullDataPath, indexPath) + if strategy == "disk" { + idx = f( + option.WithTypeName(getTypeFQN(t)), + option.WithIndexBy(indexBy), + option.WithFilesDir(path.Join(i.newConfig.Repo.Disk.Path, entityDirName)), + option.WithDataDir(i.newConfig.Repo.Disk.Path), + ) + } else if strategy == "cs3" { + idx = f( + option.WithTypeName(getTypeFQN(t)), + option.WithIndexBy(indexBy), + option.WithFilesDir(path.Join(i.newConfig.Repo.Disk.Path, entityDirName)), + option.WithDataDir(i.newConfig.Repo.Disk.Path), + option.WithDataURL(i.newConfig.Repo.CS3.DataURL), + option.WithDataPrefix(i.newConfig.Repo.CS3.DataPrefix), + option.WithJWTSecret(i.newConfig.Repo.CS3.JWTSecret), + ) + } - i.indices.addIndex(typeName, pkName, idx) + i.indices.addIndex(getTypeFQN(t), pkName, idx) return idx.Init() } func (i Indexer) AddNonUniqueIndex(t interface{}, indexBy, pkName, entityDirName string) error { - typeName := getTypeFQN(t) - fullDataPath := path.Join(i.config.DataDir, entityDirName) - indexPath := path.Join(i.config.DataDir, i.config.IndexRootDirName) + strategy := getRegistryStrategy(i.newConfig) + f := registry.IndexConstructorRegistry[strategy]["non_unique"] + idx := f( + option.WithTypeName(getTypeFQN(t)), + option.WithIndexBy(indexBy), + option.WithFilesDir(path.Join(i.config.DataDir, entityDirName)), + option.WithIndexBaseDir(path.Join(i.config.DataDir, i.config.IndexRootDirName)), + ) - idx := disk.NewNonUniqueIndex(typeName, indexBy, fullDataPath, indexPath) - - i.indices.addIndex(typeName, pkName, idx) + i.indices.addIndex(getTypeFQN(t), pkName, idx) return idx.Init() } diff --git a/accounts/pkg/indexer/indexer_test.go b/accounts/pkg/indexer/indexer_test.go index b85d8bcca2..a213fab0dd 100644 --- a/accounts/pkg/indexer/indexer_test.go +++ b/accounts/pkg/indexer/indexer_test.go @@ -1,6 +1,9 @@ package indexer import ( + "github.com/owncloud/ocis/accounts/pkg/config" + _ "github.com/owncloud/ocis/accounts/pkg/indexer/index/cs3" + _ "github.com/owncloud/ocis/accounts/pkg/indexer/index/disk" . "github.com/owncloud/ocis/accounts/pkg/indexer/test" "github.com/rs/zerolog" "github.com/stretchr/testify/assert" @@ -9,10 +12,32 @@ import ( func TestIndexer_AddWithUniqueIndex(t *testing.T) { dataDir := WriteIndexTestData(t, TestData, "Id") - indexer := NewIndex(&Config{ - DataDir: dataDir, - IndexRootDirName: "index.disk", - Log: zerolog.Logger{}, + indexer := CreateIndexer(&config.Config{ + Repo: config.Repo{ + Disk: config.Disk{ + Path: dataDir, + }, + }, + }) + + indexer.AddUniqueIndex(&User{}, "UserName", "Id", "users") + + u := &User{Id: "abcdefg-123", UserName: "mikey", Email: "mikey@example.com"} + err := indexer.Add(u) + assert.NoError(t, err) +} + +func TestIndexer_AddWithUniqueIndexCS3(t *testing.T) { + _ = WriteIndexTestDataCS3(t, TestData, "Id") + indexer := CreateIndexer(&config.Config{ + Repo: config.Repo{ + CS3: config.CS3{ + ProviderAddr: "0.0.0.0:9215", + DataURL: "http://localhost:9216", + DataPrefix: "data", + JWTSecret: "Pive-Fumkiu4", + }, + }, }) indexer.AddUniqueIndex(&User{}, "UserName", "Id", "users") @@ -21,11 +46,12 @@ func TestIndexer_AddWithUniqueIndex(t *testing.T) { err := indexer.Add(u) assert.NoError(t, err) + //_ = os.RemoveAll(dataDir) } func TestIndexer_FindByWithUniqueIndex(t *testing.T) { dataDir := WriteIndexTestData(t, TestData, "Id") - indexer := NewIndex(&Config{ + indexer := NewIndexer(&Config{ DataDir: dataDir, IndexRootDirName: "index.disk", Log: zerolog.Logger{}, @@ -44,7 +70,7 @@ func TestIndexer_FindByWithUniqueIndex(t *testing.T) { func TestIndexer_AddWithNonUniqueIndex(t *testing.T) { dataDir := WriteIndexTestData(t, TestData, "Id") - indexer := NewIndex(&Config{ + indexer := NewIndexer(&Config{ DataDir: dataDir, IndexRootDirName: "index.disk", Log: zerolog.Logger{}, @@ -69,7 +95,7 @@ func TestIndexer_AddWithNonUniqueIndex(t *testing.T) { func TestIndexer_DeleteWithNonUniqueIndex(t *testing.T) { dataDir := WriteIndexTestData(t, TestData, "Id") - indexer := NewIndex(&Config{ + indexer := NewIndexer(&Config{ DataDir: dataDir, IndexRootDirName: "index.disk", Log: zerolog.Logger{}, @@ -92,7 +118,7 @@ func TestIndexer_DeleteWithNonUniqueIndex(t *testing.T) { func TestIndexer_SearchWithNonUniqueIndex(t *testing.T) { dataDir := WriteIndexTestData(t, TestData, "Id") - indexer := NewIndex(&Config{ + indexer := NewIndexer(&Config{ DataDir: dataDir, IndexRootDirName: "index.disk", Log: zerolog.Logger{}, @@ -117,7 +143,7 @@ func TestIndexer_SearchWithNonUniqueIndex(t *testing.T) { func TestIndexer_UpdateWithUniqueIndex(t *testing.T) { dataDir := WriteIndexTestData(t, TestData, "Id") - indexer := NewIndex(&Config{ + indexer := NewIndexer(&Config{ DataDir: dataDir, IndexRootDirName: "index.disk", Log: zerolog.Logger{}, @@ -171,7 +197,7 @@ func TestIndexer_UpdateWithUniqueIndex(t *testing.T) { func TestIndexer_UpdateWithNonUniqueIndex(t *testing.T) { dataDir := WriteIndexTestData(t, TestData, "Id") - indexer := NewIndex(&Config{ + indexer := NewIndexer(&Config{ DataDir: dataDir, IndexRootDirName: "index.disk", Log: zerolog.Logger{}, @@ -192,7 +218,7 @@ func TestIndexer_UpdateWithNonUniqueIndex(t *testing.T) { /* func TestManagerQueryMultipleIndices(t *testing.T) { dataDir := writeIndexTestData(t, testData, "Id") - man := NewIndex(&Config{ + man := NewIndexer(&Config{ DataDir: dataDir, IndexRootDirName: "index.disk", Log: zerolog.Logger{}, @@ -246,7 +272,7 @@ func TestManagerQueryMultipleIndices(t *testing.T) { /* func TestManagerDelete(t *testing.T) { dataDir := writeIndexTestData(t, testData, "Id") - man := NewIndex(&Config{ + man := NewIndexer(&Config{ DataDir: dataDir, IndexRootDirName: "index.disk", Log: zerolog.Logger{}, diff --git a/accounts/pkg/indexer/option/option.go b/accounts/pkg/indexer/option/option.go new file mode 100644 index 0000000000..aa6a8f42b0 --- /dev/null +++ b/accounts/pkg/indexer/option/option.go @@ -0,0 +1,85 @@ +package option + +// Option defines a single option function. +type Option func(o *Options) + +// Options defines the available options for this package. +type Options struct { + // Disk Options + TypeName string + IndexBy string + FilesDir string + IndexBaseDir string + DataDir string + EntityDirName string + + // CS3 options + DataURL string + DataPrefix string + JWTSecret string +} + +// newOptions initializes the available default options. +func newOptions(opts ...Option) Options { + opt := Options{} + + for _, o := range opts { + o(&opt) + } + + return opt +} + +func WithJWTSecret(val string) Option { + return func(o *Options) { + o.JWTSecret = val + } +} + +func WithDataURL(val string) Option { + return func(o *Options) { + o.DataURL = val + } +} + +func WithDataPrefix(val string) Option { + return func(o *Options) { + o.DataPrefix = val + } +} + +func WithEntityDirName(val string) Option { + return func(o *Options) { + o.EntityDirName = val + } +} + +func WithDataDir(val string) Option { + return func(o *Options) { + o.DataDir = val + } +} + +func WithTypeName(val string) Option { + return func(o *Options) { + o.TypeName = val + } +} + +func WithIndexBy(val string) Option { + return func(o *Options) { + o.IndexBy = val + } +} + +func WithIndexBaseDir(val string) Option { + return func(o *Options) { + o.IndexBaseDir = val + } +} + +func WithFilesDir(val string) Option { + return func(o *Options) { + o.FilesDir = val + } +} diff --git a/accounts/pkg/indexer/registry/registry.go b/accounts/pkg/indexer/registry/registry.go new file mode 100644 index 0000000000..56c6882df7 --- /dev/null +++ b/accounts/pkg/indexer/registry/registry.go @@ -0,0 +1,14 @@ +package registry + +import ( + "github.com/owncloud/ocis/accounts/pkg/indexer/index" + "github.com/owncloud/ocis/accounts/pkg/indexer/option" +) + +type IndexConstructor func(o ...option.Option) index.Index + +// IndexConstructorRegistry undocumented. +var IndexConstructorRegistry = map[string]map[string]IndexConstructor{ + "disk": {}, + "cs3": {}, +}