mirror of
https://github.com/opencloud-eu/opencloud.git
synced 2026-02-23 05:59:28 -06:00
first draft implementation of an autoincremental index
This commit is contained in:
A.Unger
186
accounts/pkg/indexer/index/disk/autoincrement.go
Normal file
186
accounts/pkg/indexer/index/disk/autoincrement.go
Normal file
@@ -0,0 +1,186 @@
|
||||
package disk
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
idxerrs "github.com/owncloud/ocis/accounts/pkg/indexer/errors"
|
||||
|
||||
"github.com/owncloud/ocis/accounts/pkg/indexer/index"
|
||||
"github.com/owncloud/ocis/accounts/pkg/indexer/option"
|
||||
"github.com/owncloud/ocis/accounts/pkg/indexer/registry"
|
||||
)
|
||||
|
||||
type Autoincrement struct {
|
||||
indexBy string
|
||||
typeName string
|
||||
filesDir string
|
||||
indexBaseDir string
|
||||
indexRootDir string
|
||||
entity interface{}
|
||||
}
|
||||
|
||||
// - Creating an autoincrement index has to be thread safe.
|
||||
// - Validation: autoincrement indexes should only work on integers.
|
||||
|
||||
func init() {
|
||||
registry.IndexConstructorRegistry["disk"]["autoincrement"] = NewAutoincrementIndex
|
||||
}
|
||||
|
||||
// NewAutoincrementIndex instantiates a new UniqueIndex instance. Init() should be
|
||||
// called afterward to ensure correct on-disk structure.
|
||||
func NewAutoincrementIndex(o ...option.Option) index.Index {
|
||||
opts := &option.Options{}
|
||||
for _, opt := range o {
|
||||
opt(opts)
|
||||
}
|
||||
|
||||
// validate the field
|
||||
if opts.Entity == nil {
|
||||
// return error: entity needed for field validation
|
||||
}
|
||||
|
||||
k, err := getKind(opts.Entity, opts.IndexBy)
|
||||
if !isValidKind(k) || err != nil {
|
||||
panic("invalid index in non-numeric field")
|
||||
}
|
||||
|
||||
return &Autoincrement{
|
||||
indexBy: opts.IndexBy,
|
||||
typeName: opts.TypeName,
|
||||
filesDir: opts.FilesDir,
|
||||
indexBaseDir: path.Join(opts.DataDir, "index.disk"),
|
||||
indexRootDir: path.Join(path.Join(opts.DataDir, "index.disk"), strings.Join([]string{"autoincrement", opts.TypeName, opts.IndexBy}, ".")),
|
||||
}
|
||||
}
|
||||
|
||||
var (
|
||||
validKinds = []reflect.Kind{
|
||||
reflect.Int,
|
||||
reflect.Int8,
|
||||
reflect.Int16,
|
||||
reflect.Int32,
|
||||
}
|
||||
)
|
||||
|
||||
func (idx Autoincrement) Init() error {
|
||||
if _, err := os.Stat(idx.filesDir); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := os.MkdirAll(idx.indexRootDir, 0777); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (idx Autoincrement) Lookup(v string) ([]string, error) {
|
||||
searchPath := path.Join(idx.indexRootDir, v)
|
||||
if err := isValidSymlink(searchPath); err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
err = &idxerrs.NotFoundErr{TypeName: idx.typeName, Key: idx.indexBy, Value: v}
|
||||
}
|
||||
|
||||
return nil, err
|
||||
}
|
||||
|
||||
p, err := os.Readlink(searchPath)
|
||||
if err != nil {
|
||||
return []string{}, nil
|
||||
}
|
||||
|
||||
return []string{p}, err
|
||||
}
|
||||
|
||||
func (idx Autoincrement) Add(id, v string) (string, error) {
|
||||
oldName := filepath.Join(idx.filesDir, id)
|
||||
newName := filepath.Join(idx.indexRootDir, strconv.Itoa(idx.next()))
|
||||
err := os.Symlink(oldName, newName)
|
||||
if errors.Is(err, os.ErrExist) {
|
||||
return "", &idxerrs.AlreadyExistsErr{TypeName: idx.typeName, Key: idx.indexBy, Value: v}
|
||||
}
|
||||
|
||||
return newName, err
|
||||
}
|
||||
|
||||
func (idx Autoincrement) Remove(id string, v string) error {
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
func (idx Autoincrement) Update(id, oldV, newV string) error {
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
func (idx Autoincrement) Search(pattern string) ([]string, error) {
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
func (idx Autoincrement) IndexBy() string {
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
func (idx Autoincrement) TypeName() string {
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
func (idx Autoincrement) FilesDir() string {
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
func isValidKind(k reflect.Kind) bool {
|
||||
for _, v := range validKinds {
|
||||
if k == v {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func getKind(i interface{}, field string) (reflect.Kind, error) {
|
||||
r := reflect.ValueOf(i)
|
||||
// TODO reflect.FieldByName panics. Recover from it.
|
||||
// further read: https://blog.golang.org/defer-panic-and-recover
|
||||
return reflect.Indirect(r).FieldByName(field).Kind(), nil
|
||||
}
|
||||
|
||||
func readDir(dirname string) ([]os.FileInfo, error) {
|
||||
f, err := os.Open(dirname)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
list, err := f.Readdir(-1)
|
||||
f.Close()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sort.Slice(list, func(i, j int) bool {
|
||||
a, _ := strconv.Atoi(list[i].Name())
|
||||
b, _ := strconv.Atoi(list[j].Name())
|
||||
return a < b
|
||||
})
|
||||
return list, nil
|
||||
}
|
||||
|
||||
func (idx Autoincrement) next() int {
|
||||
files, err := readDir(idx.indexRootDir)
|
||||
if err != nil {
|
||||
// hello handle me pls.
|
||||
}
|
||||
|
||||
if len(files) == 0 {
|
||||
return 0
|
||||
}
|
||||
|
||||
latest, err := strconv.Atoi(path.Base(files[len(files)-1].Name())) // would returning a string be a better interface?
|
||||
if err != nil {
|
||||
// handle me daddy
|
||||
}
|
||||
return latest + 1
|
||||
}
|
||||
193
accounts/pkg/indexer/index/disk/autoincrement_test.go
Normal file
193
accounts/pkg/indexer/index/disk/autoincrement_test.go
Normal file
@@ -0,0 +1,193 @@
|
||||
package disk
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/owncloud/ocis/accounts/pkg/indexer/option"
|
||||
//. "github.com/owncloud/ocis/accounts/pkg/indexer/test"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestIsValidKind(t *testing.T) {
|
||||
scenarios := []struct {
|
||||
panics bool
|
||||
name string
|
||||
indexBy string
|
||||
entity struct {
|
||||
Number int
|
||||
Name string
|
||||
NumberFloat float32
|
||||
}
|
||||
}{
|
||||
{
|
||||
name: "valid autoincrement index",
|
||||
panics: false,
|
||||
indexBy: "Number",
|
||||
entity: struct {
|
||||
Number int
|
||||
Name string
|
||||
NumberFloat float32
|
||||
}{
|
||||
Name: "tesy-mc-testace",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "create autoincrement index on a non-existing field",
|
||||
panics: true,
|
||||
indexBy: "Age",
|
||||
entity: struct {
|
||||
Number int
|
||||
Name string
|
||||
NumberFloat float32
|
||||
}{
|
||||
Name: "tesy-mc-testace",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "attempt to create an autoincrement index with no entity",
|
||||
panics: true,
|
||||
indexBy: "Age",
|
||||
},
|
||||
{
|
||||
name: "create autoincrement index on a non-numeric field",
|
||||
panics: true,
|
||||
indexBy: "Name",
|
||||
entity: struct {
|
||||
Number int
|
||||
Name string
|
||||
NumberFloat float32
|
||||
}{
|
||||
Name: "tesy-mc-testace",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, scenario := range scenarios {
|
||||
t.Run(scenario.name, func(t *testing.T) {
|
||||
if scenario.panics {
|
||||
assert.Panics(t, func() {
|
||||
_ = NewAutoincrementIndex(
|
||||
option.WithEntity(scenario.entity),
|
||||
option.WithIndexBy(scenario.indexBy),
|
||||
)
|
||||
})
|
||||
} else {
|
||||
assert.NotPanics(t, func() {
|
||||
_ = NewAutoincrementIndex(
|
||||
option.WithEntity(scenario.entity),
|
||||
option.WithIndexBy(scenario.indexBy),
|
||||
)
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNext(t *testing.T) {
|
||||
scenarios := []struct {
|
||||
name string
|
||||
expected int
|
||||
indexBy string
|
||||
entity interface{}
|
||||
}{
|
||||
{
|
||||
name: "get next value",
|
||||
expected: 0,
|
||||
indexBy: "Number",
|
||||
entity: struct {
|
||||
Number int
|
||||
Name string
|
||||
NumberFloat float32
|
||||
}{
|
||||
Name: "tesy-mc-testace",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, scenario := range scenarios {
|
||||
t.Run(scenario.name, func(t *testing.T) {
|
||||
tmpDir, err := createTmpDirStr()
|
||||
assert.NoError(t, err)
|
||||
|
||||
err = os.MkdirAll(filepath.Join(tmpDir, "data"), 0777)
|
||||
assert.NoError(t, err)
|
||||
|
||||
i := NewAutoincrementIndex(
|
||||
option.WithDataDir(tmpDir),
|
||||
option.WithFilesDir(filepath.Join(tmpDir, "data")),
|
||||
option.WithEntity(scenario.entity),
|
||||
option.WithTypeName("LambdaType"),
|
||||
option.WithIndexBy(scenario.indexBy),
|
||||
)
|
||||
|
||||
err = i.Init()
|
||||
assert.NoError(t, err)
|
||||
|
||||
tmpFile, err := os.Create(filepath.Join(tmpDir, "data", "test-example"))
|
||||
assert.NoError(t, err)
|
||||
assert.NoError(t, tmpFile.Close())
|
||||
|
||||
oldName, err := i.Add("test-example", "")
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, filepath.Base(oldName), 0)
|
||||
|
||||
oldName, err = i.Add("test-example", "")
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, filepath.Base(oldName), 1)
|
||||
|
||||
oldName, err = i.Add("test-example", "")
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, filepath.Base(oldName), 2)
|
||||
t.Log(oldName)
|
||||
|
||||
_ = os.RemoveAll(tmpDir)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkAdd(b *testing.B) {
|
||||
tmpDir, err := createTmpDirStr()
|
||||
assert.NoError(b, err)
|
||||
|
||||
err = os.MkdirAll(filepath.Join(tmpDir, "data"), 0777)
|
||||
assert.NoError(b, err)
|
||||
|
||||
tmpFile, err := os.Create(filepath.Join(tmpDir, "data", "test-example"))
|
||||
assert.NoError(b, err)
|
||||
assert.NoError(b, tmpFile.Close())
|
||||
|
||||
i := NewAutoincrementIndex(
|
||||
option.WithDataDir(tmpDir),
|
||||
option.WithFilesDir(filepath.Join(tmpDir, "data")),
|
||||
option.WithEntity(struct {
|
||||
Number int
|
||||
Name string
|
||||
NumberFloat float32
|
||||
}{}),
|
||||
option.WithTypeName("LambdaType"),
|
||||
option.WithIndexBy("Number"),
|
||||
)
|
||||
|
||||
err = i.Init()
|
||||
assert.NoError(b, err)
|
||||
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := i.Add("test-example", "")
|
||||
if err != nil {
|
||||
b.Error(err)
|
||||
}
|
||||
assert.NoError(b, err)
|
||||
}
|
||||
}
|
||||
|
||||
func createTmpDirStr() (string, error) {
|
||||
name, err := ioutil.TempDir("/var/tmp", "testfiles-*")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return name, nil
|
||||
}
|
||||
@@ -12,6 +12,7 @@ type Options struct {
|
||||
IndexBaseDir string
|
||||
DataDir string
|
||||
EntityDirName string
|
||||
Entity interface{}
|
||||
|
||||
// CS3 options
|
||||
DataURL string
|
||||
@@ -20,6 +21,13 @@ type Options struct {
|
||||
ProviderAddr string
|
||||
}
|
||||
|
||||
// WithEntity sets the JWTSecret field.
|
||||
func WithEntity(val interface{}) Option {
|
||||
return func(o *Options) {
|
||||
o.Entity = val
|
||||
}
|
||||
}
|
||||
|
||||
// WithJWTSecret sets the JWTSecret field.
|
||||
func WithJWTSecret(val string) Option {
|
||||
return func(o *Options) {
|
||||
|
||||
Reference in New Issue
Block a user