mirror/opencloud
1
0
Fork 0
mirror of https://github.com/opencloud-eu/opencloud.git synced 2026-01-25 05:58:38 -06:00
Code Issues Packages Projects Releases Wiki Activity

wip config

Browse Source
This commit is contained in:
A.Unger
2020-10-06 16:57:19 +02:00
committed by Ilja Neumann
parent e32738e812
commit a852edc763
8 changed files with 280 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
accounts/pkg/config/config.go
View File

@@ -77,11 +77,12 @@ type CS3 struct {
ProviderAddr string
DataURL string
DataPrefix string
JWTSecret string
}
// ServiceUser defines the user required for EOS
type ServiceUser struct {
UUID string
UUID string
Username string
UID int64
GID int64

35
accounts/pkg/indexer/index/cs3/unique.go
View File

@@ -10,6 +10,9 @@ import (
"github.com/cs3org/reva/pkg/token"
"github.com/cs3org/reva/pkg/token/manager/jwt"
idxerrs "github.com/owncloud/ocis/accounts/pkg/indexer/errors"
"github.com/owncloud/ocis/accounts/pkg/indexer/index"
"github.com/owncloud/ocis/accounts/pkg/indexer/option"
"github.com/owncloud/ocis/accounts/pkg/indexer/registry"
"google.golang.org/grpc/metadata"
"io/ioutil"
"net/http"
@@ -42,6 +45,38 @@ type Config struct {
ServiceUserUUID string
}
func init() {
registry.IndexConstructorRegistry["cs3"]["unique"] = NewUniqueIndexWithOptions
}
// NewUniqueIndexWithOptions instantiates a new UniqueIndex instance. Init() should be
// called afterward to ensure correct on-disk structure.
func NewUniqueIndexWithOptions(o ...option.Option) index.Index {
opts := &option.Options{}
for _, opt := range o {
opt(opts)
}
u := &Unique{
indexBy: opts.IndexBy,
typeName: opts.TypeName,
filesDir: opts.FilesDir,
indexBaseDir: path.Join(opts.DataDir, "index.cs3"),
indexRootDir: path.Join(path.Join(opts.DataDir, "index.cs3"), strings.Join([]string{"unique", opts.TypeName, opts.IndexBy}, ".")),
cs3conf: &Config{
JWTSecret: opts.JWTSecret,
},
dataProvider: dataProviderClient{
baseURL: singleJoiningSlash(opts.DataURL, opts.DataPrefix),
client: http.Client{
Transport: http.DefaultTransport,
},
},
}
return u
}
// NewUniqueIndex instantiates a new UniqueIndex instance. Init() should be
// called afterward to ensure correct on-disk structure.
func NewUniqueIndex(typeName, indexBy, filesDir, indexBaseDir string, cfg *Config) Unique {

25
accounts/pkg/indexer/index/disk/non_unique.go
View File

@@ -4,10 +4,14 @@ import (
"errors"
"fmt"
idxerrs "github.com/owncloud/ocis/accounts/pkg/indexer/errors"
"github.com/owncloud/ocis/accounts/pkg/indexer/index"
"github.com/owncloud/ocis/accounts/pkg/indexer/option"
"github.com/owncloud/ocis/accounts/pkg/indexer/registry"
"io/ioutil"
"os"
"path"
"path/filepath"
"strings"
)
// NonUniqueIndex is able to index an document by a key which might contain non-unique values
@@ -28,6 +32,27 @@ type NonUniqueIndex struct {
indexRootDir string
}
func init() {
registry.IndexConstructorRegistry["disk"]["non_unique"] = NewNonUniqueIndexWithOptions
}
// NewNonUniqueIndexWithOptions instantiates a new UniqueIndex instance. Init() should be
// called afterward to ensure correct on-disk structure.
func NewNonUniqueIndexWithOptions(o ...option.Option) index.Index {
opts := &option.Options{}
for _, opt := range o {
opt(opts)
}
return NonUniqueIndex{
indexBy: opts.IndexBy,
typeName: opts.TypeName,
filesDir: opts.FilesDir,
indexBaseDir: opts.IndexBaseDir,
indexRootDir: path.Join(opts.IndexBaseDir, strings.Join([]string{"unique", opts.TypeName, opts.IndexBy}, ".")),
}
}
// NewNonUniqueIndex instantiates a new NonUniqueIndex instance. Init() should be
// called afterward to ensure correct on-disk structure.
func NewNonUniqueIndex(typeName, indexBy, filesDir, indexBaseDir string) NonUniqueIndex {

28
accounts/pkg/indexer/index/disk/unique.go
View File

@@ -4,6 +4,9 @@ import (
"errors"
"fmt"
idxerrs "github.com/owncloud/ocis/accounts/pkg/indexer/errors"
"github.com/owncloud/ocis/accounts/pkg/indexer/index"
"github.com/owncloud/ocis/accounts/pkg/indexer/option"
"github.com/owncloud/ocis/accounts/pkg/indexer/registry"
"os"
"path"
"path/filepath"
@@ -38,6 +41,29 @@ type Unique struct {
indexRootDir string
}
func init() {
registry.IndexConstructorRegistry["disk"]["unique"] = NewUniqueIndexWithOptions
}
// NewUniqueIndexWithOptions instantiates a new UniqueIndex instance. Init() should be
// called afterward to ensure correct on-disk structure.
func NewUniqueIndexWithOptions(o ...option.Option) index.Index {
opts := &option.Options{}
for _, opt := range o {
opt(opts)
}
u := &Unique{
indexBy: opts.IndexBy,
typeName: opts.TypeName,
filesDir: opts.FilesDir,
indexBaseDir: path.Join(opts.DataDir, "index.disk"),
indexRootDir: path.Join(path.Join(opts.DataDir, "index.disk"), strings.Join([]string{"unique", opts.TypeName, opts.IndexBy}, ".")),
}
return u
}
// NewUniqueIndex instantiates a new UniqueIndex instance. Init() should be
// called afterward to ensure correct on-disk structure.
func NewUniqueIndex(typeName, indexBy, filesDir, indexBaseDir string) Unique {
@@ -50,7 +76,7 @@ func NewUniqueIndex(typeName, indexBy, filesDir, indexBaseDir string) Unique {
}
}
func (idx Unique) Init() error {
func (idx *Unique) Init() error {
if _, err := os.Stat(idx.filesDir); err != nil {
return err
}

69
accounts/pkg/indexer/indexer.go
View File

@@ -3,16 +3,20 @@ package indexer
import (
"fmt"
"github.com/owncloud/ocis/accounts/pkg/config"
"github.com/owncloud/ocis/accounts/pkg/indexer/errors"
"github.com/owncloud/ocis/accounts/pkg/indexer/index/disk"
"github.com/owncloud/ocis/accounts/pkg/indexer/index"
"github.com/owncloud/ocis/accounts/pkg/indexer/option"
"github.com/owncloud/ocis/accounts/pkg/indexer/registry"
"github.com/rs/zerolog"
"path"
)
// Indexer is a facade to configure and query over multiple indices.
type Indexer struct {
config *Config
indices typeMap
newConfig *config.Config
config *Config
indices typeMap
}
type Config struct {
@@ -21,32 +25,67 @@ type Config struct {
Log zerolog.Logger
}
func NewIndex(cfg *Config) *Indexer {
func NewIndexer(cfg *Config) *Indexer {
return &Indexer{
config: cfg,
indices: typeMap{},
}
}
func CreateIndexer(cfg *config.Config) *Indexer {
return &Indexer{
newConfig: cfg,
indices: typeMap{},
}
}
func getRegistryStrategy(cfg *config.Config) string {
if cfg.Repo.Disk.Path != "" {
return "disk"
}
return "cs3"
}
func (i Indexer) AddUniqueIndex(t interface{}, indexBy, pkName, entityDirName string) error {
typeName := getTypeFQN(t)
fullDataPath := path.Join(i.config.DataDir, entityDirName)
indexPath := path.Join(i.config.DataDir, i.config.IndexRootDirName)
strategy := getRegistryStrategy(i.newConfig)
f := registry.IndexConstructorRegistry[strategy]["unique"]
var idx index.Index
idx := disk.NewUniqueIndex(typeName, indexBy, fullDataPath, indexPath)
if strategy == "disk" {
idx = f(
option.WithTypeName(getTypeFQN(t)),
option.WithIndexBy(indexBy),
option.WithFilesDir(path.Join(i.newConfig.Repo.Disk.Path, entityDirName)),
option.WithDataDir(i.newConfig.Repo.Disk.Path),
)
} else if strategy == "cs3" {
idx = f(
option.WithTypeName(getTypeFQN(t)),
option.WithIndexBy(indexBy),
option.WithFilesDir(path.Join(i.newConfig.Repo.Disk.Path, entityDirName)),
option.WithDataDir(i.newConfig.Repo.Disk.Path),
option.WithDataURL(i.newConfig.Repo.CS3.DataURL),
option.WithDataPrefix(i.newConfig.Repo.CS3.DataPrefix),
option.WithJWTSecret(i.newConfig.Repo.CS3.JWTSecret),
)
}
i.indices.addIndex(typeName, pkName, idx)
i.indices.addIndex(getTypeFQN(t), pkName, idx)
return idx.Init()
}
func (i Indexer) AddNonUniqueIndex(t interface{}, indexBy, pkName, entityDirName string) error {
typeName := getTypeFQN(t)
fullDataPath := path.Join(i.config.DataDir, entityDirName)
indexPath := path.Join(i.config.DataDir, i.config.IndexRootDirName)
strategy := getRegistryStrategy(i.newConfig)
f := registry.IndexConstructorRegistry[strategy]["non_unique"]
idx := f(
option.WithTypeName(getTypeFQN(t)),
option.WithIndexBy(indexBy),
option.WithFilesDir(path.Join(i.config.DataDir, entityDirName)),
option.WithIndexBaseDir(path.Join(i.config.DataDir, i.config.IndexRootDirName)),
)
idx := disk.NewNonUniqueIndex(typeName, indexBy, fullDataPath, indexPath)
i.indices.addIndex(typeName, pkName, idx)
i.indices.addIndex(getTypeFQN(t), pkName, idx)
return idx.Init()
}

50
accounts/pkg/indexer/indexer_test.go
View File

@@ -1,6 +1,9 @@
package indexer
import (
"github.com/owncloud/ocis/accounts/pkg/config"
_ "github.com/owncloud/ocis/accounts/pkg/indexer/index/cs3"
_ "github.com/owncloud/ocis/accounts/pkg/indexer/index/disk"
. "github.com/owncloud/ocis/accounts/pkg/indexer/test"
"github.com/rs/zerolog"
"github.com/stretchr/testify/assert"
@@ -9,10 +12,32 @@ import (
func TestIndexer_AddWithUniqueIndex(t *testing.T) {
dataDir := WriteIndexTestData(t, TestData, "Id")
indexer := NewIndex(&Config{
DataDir: dataDir,
IndexRootDirName: "index.disk",
Log: zerolog.Logger{},
indexer := CreateIndexer(&config.Config{
Repo: config.Repo{
Disk: config.Disk{
Path: dataDir,
},
},
})
indexer.AddUniqueIndex(&User{}, "UserName", "Id", "users")
u := &User{Id: "abcdefg-123", UserName: "mikey", Email: "mikey@example.com"}
err := indexer.Add(u)
assert.NoError(t, err)
}
func TestIndexer_AddWithUniqueIndexCS3(t *testing.T) {
_ = WriteIndexTestDataCS3(t, TestData, "Id")
indexer := CreateIndexer(&config.Config{
Repo: config.Repo{
CS3: config.CS3{
ProviderAddr: "0.0.0.0:9215",
DataURL: "http://localhost:9216",
DataPrefix: "data",
JWTSecret: "Pive-Fumkiu4",
},
},
})
indexer.AddUniqueIndex(&User{}, "UserName", "Id", "users")
@@ -21,11 +46,12 @@ func TestIndexer_AddWithUniqueIndex(t *testing.T) {
err := indexer.Add(u)
assert.NoError(t, err)
//_ = os.RemoveAll(dataDir)
}
func TestIndexer_FindByWithUniqueIndex(t *testing.T) {
dataDir := WriteIndexTestData(t, TestData, "Id")
indexer := NewIndex(&Config{
indexer := NewIndexer(&Config{
DataDir: dataDir,
IndexRootDirName: "index.disk",
Log: zerolog.Logger{},
@@ -44,7 +70,7 @@ func TestIndexer_FindByWithUniqueIndex(t *testing.T) {
func TestIndexer_AddWithNonUniqueIndex(t *testing.T) {
dataDir := WriteIndexTestData(t, TestData, "Id")
indexer := NewIndex(&Config{
indexer := NewIndexer(&Config{
DataDir: dataDir,
IndexRootDirName: "index.disk",
Log: zerolog.Logger{},
@@ -69,7 +95,7 @@ func TestIndexer_AddWithNonUniqueIndex(t *testing.T) {
func TestIndexer_DeleteWithNonUniqueIndex(t *testing.T) {
dataDir := WriteIndexTestData(t, TestData, "Id")
indexer := NewIndex(&Config{
indexer := NewIndexer(&Config{
DataDir: dataDir,
IndexRootDirName: "index.disk",
Log: zerolog.Logger{},
@@ -92,7 +118,7 @@ func TestIndexer_DeleteWithNonUniqueIndex(t *testing.T) {
func TestIndexer_SearchWithNonUniqueIndex(t *testing.T) {
dataDir := WriteIndexTestData(t, TestData, "Id")
indexer := NewIndex(&Config{
indexer := NewIndexer(&Config{
DataDir: dataDir,
IndexRootDirName: "index.disk",
Log: zerolog.Logger{},
@@ -117,7 +143,7 @@ func TestIndexer_SearchWithNonUniqueIndex(t *testing.T) {
func TestIndexer_UpdateWithUniqueIndex(t *testing.T) {
dataDir := WriteIndexTestData(t, TestData, "Id")
indexer := NewIndex(&Config{
indexer := NewIndexer(&Config{
DataDir: dataDir,
IndexRootDirName: "index.disk",
Log: zerolog.Logger{},
@@ -171,7 +197,7 @@ func TestIndexer_UpdateWithUniqueIndex(t *testing.T) {
func TestIndexer_UpdateWithNonUniqueIndex(t *testing.T) {
dataDir := WriteIndexTestData(t, TestData, "Id")
indexer := NewIndex(&Config{
indexer := NewIndexer(&Config{
DataDir: dataDir,
IndexRootDirName: "index.disk",
Log: zerolog.Logger{},
@@ -192,7 +218,7 @@ func TestIndexer_UpdateWithNonUniqueIndex(t *testing.T) {
/*
func TestManagerQueryMultipleIndices(t *testing.T) {
dataDir := writeIndexTestData(t, testData, "Id")
man := NewIndex(&Config{
man := NewIndexer(&Config{
DataDir: dataDir,
IndexRootDirName: "index.disk",
Log: zerolog.Logger{},
@@ -246,7 +272,7 @@ func TestManagerQueryMultipleIndices(t *testing.T) {
/*
func TestManagerDelete(t *testing.T) {
dataDir := writeIndexTestData(t, testData, "Id")
man := NewIndex(&Config{
man := NewIndexer(&Config{
DataDir: dataDir,
IndexRootDirName: "index.disk",
Log: zerolog.Logger{},

85
accounts/pkg/indexer/option/option.go Normal file
View File

@@ -0,0 +1,85 @@
package option
// Option defines a single option function.
type Option func(o *Options)
// Options defines the available options for this package.
type Options struct {
// Disk Options
TypeName string
IndexBy string
FilesDir string
IndexBaseDir string
DataDir string
EntityDirName string
// CS3 options
DataURL string
DataPrefix string
JWTSecret string
}
// newOptions initializes the available default options.
func newOptions(opts ...Option) Options {
opt := Options{}
for _, o := range opts {
o(&opt)
}
return opt
}
func WithJWTSecret(val string) Option {
return func(o *Options) {
o.JWTSecret = val
}
}
func WithDataURL(val string) Option {
return func(o *Options) {
o.DataURL = val
}
}
func WithDataPrefix(val string) Option {
return func(o *Options) {
o.DataPrefix = val
}
}
func WithEntityDirName(val string) Option {
return func(o *Options) {
o.EntityDirName = val
}
}
func WithDataDir(val string) Option {
return func(o *Options) {
o.DataDir = val
}
}
func WithTypeName(val string) Option {
return func(o *Options) {
o.TypeName = val
}
}
func WithIndexBy(val string) Option {
return func(o *Options) {
o.IndexBy = val
}
}
func WithIndexBaseDir(val string) Option {
return func(o *Options) {
o.IndexBaseDir = val
}
}
func WithFilesDir(val string) Option {
return func(o *Options) {
o.FilesDir = val
}
}

14
accounts/pkg/indexer/registry/registry.go Normal file
View File

@@ -0,0 +1,14 @@
package registry
import (
"github.com/owncloud/ocis/accounts/pkg/indexer/index"
"github.com/owncloud/ocis/accounts/pkg/indexer/option"
)
type IndexConstructor func(o ...option.Option) index.Index
// IndexConstructorRegistry undocumented.
var IndexConstructorRegistry = map[string]map[string]IndexConstructor{
"disk": {},
"cs3": {},
}