mirror of
https://github.com/opencloud-eu/opencloud.git
synced 2026-02-22 21:49:07 -06:00
Implement Add for cs3 autoincrement index
This commit is contained in:
Benedikt Kulmann
committed by
A.Unger
A.Unger
parent
5bdb94e92d
commit
8abdb833d4
@@ -3,10 +3,13 @@ package cs3
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
idxerrs "github.com/owncloud/ocis/accounts/pkg/indexer/errors"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"os"
|
||||
"path"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
|
||||
@@ -113,7 +116,20 @@ func (idx AutoincrementIndex) Lookup(v string) ([]string, error) {
|
||||
}
|
||||
|
||||
func (idx AutoincrementIndex) Add(id, v string) (string, error) {
|
||||
panic("implement me")
|
||||
next, err := idx.next()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
newName := path.Join(idx.indexRootDir, strconv.Itoa(next))
|
||||
if err := idx.createSymlink(id, newName); err != nil {
|
||||
if os.IsExist(err) {
|
||||
return "", &idxerrs.AlreadyExistsErr{TypeName: idx.typeName, Key: idx.indexBy, Value: v}
|
||||
}
|
||||
|
||||
return "", err
|
||||
}
|
||||
|
||||
return newName, nil
|
||||
}
|
||||
|
||||
func (idx AutoincrementIndex) Remove(id string, v string) error {
|
||||
@@ -222,3 +238,39 @@ func (idx *AutoincrementIndex) authenticate(ctx context.Context) (token string,
|
||||
}
|
||||
return idx.tokenManager.MintToken(ctx, u)
|
||||
}
|
||||
|
||||
func (idx AutoincrementIndex) next() (int, error) {
|
||||
ctx := context.Background()
|
||||
t, err := idx.authenticate(ctx)
|
||||
if err != nil {
|
||||
return -1, err
|
||||
}
|
||||
|
||||
ctx = metadata.AppendToOutgoingContext(ctx, token.TokenHeader, t)
|
||||
res, err := idx.storageProvider.ListContainer(ctx, &provider.ListContainerRequest{
|
||||
Ref: &provider.Reference{
|
||||
Spec: &provider.Reference_Path{Path: path.Join("/meta", idx.indexRootDir)},
|
||||
},
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return -1, err
|
||||
}
|
||||
|
||||
if len(res.GetInfos()) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
infos := res.GetInfos()
|
||||
sort.Slice(infos, func(i, j int) bool {
|
||||
a, _ := strconv.Atoi(path.Base(infos[i].Path))
|
||||
b, _ := strconv.Atoi(path.Base(infos[j].Path))
|
||||
return a < b
|
||||
})
|
||||
|
||||
latest, err := strconv.Atoi(path.Base(infos[len(infos)-1].Path)) // would returning a string be a better interface?
|
||||
if err != nil {
|
||||
return -1, err
|
||||
}
|
||||
return latest + 1, nil
|
||||
}
|
||||
|
||||
@@ -1,5 +1,87 @@
|
||||
package cs3
|
||||
|
||||
func TestAutoincrementIndexAdd() {
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/owncloud/ocis/accounts/pkg/config"
|
||||
"github.com/owncloud/ocis/accounts/pkg/indexer/option"
|
||||
. "github.com/owncloud/ocis/accounts/pkg/indexer/test"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestAutoincrementIndexAdd(t *testing.T) {
|
||||
dataDir := WriteIndexTestDataCS3(t, Data, "ID")
|
||||
cfg := config.Config{
|
||||
Repo: config.Repo{
|
||||
Disk: config.Disk{
|
||||
Path: "",
|
||||
},
|
||||
CS3: config.CS3{
|
||||
ProviderAddr: "0.0.0.0:9215",
|
||||
DataURL: "http://localhost:9216",
|
||||
DataPrefix: "data",
|
||||
JWTSecret: "Pive-Fumkiu4",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
sut := NewAutoincrementIndex(
|
||||
option.WithTypeName(GetTypeFQN(User{})),
|
||||
option.WithIndexBy("UID"),
|
||||
option.WithDataURL(cfg.Repo.CS3.DataURL),
|
||||
option.WithDataPrefix(cfg.Repo.CS3.DataPrefix),
|
||||
option.WithJWTSecret(cfg.Repo.CS3.JWTSecret),
|
||||
option.WithProviderAddr(cfg.Repo.CS3.ProviderAddr),
|
||||
)
|
||||
|
||||
err := sut.Init()
|
||||
assert.NoError(t, err)
|
||||
|
||||
for i := 0; i < 5; i++ {
|
||||
res, err := sut.Add("abcdefg-123", "ignored")
|
||||
assert.NoError(t, err)
|
||||
t.Log(res)
|
||||
}
|
||||
|
||||
_ = os.RemoveAll(dataDir)
|
||||
}
|
||||
|
||||
func BenchmarkAutoincrementIndexAdd(b *testing.B) {
|
||||
dataDir := WriteIndexBenchmarkDataCS3(b, Data, "ID")
|
||||
cfg := config.Config{
|
||||
Repo: config.Repo{
|
||||
Disk: config.Disk{
|
||||
Path: "",
|
||||
},
|
||||
CS3: config.CS3{
|
||||
ProviderAddr: "0.0.0.0:9215",
|
||||
DataURL: "http://localhost:9216",
|
||||
DataPrefix: "data",
|
||||
JWTSecret: "Pive-Fumkiu4",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
sut := NewAutoincrementIndex(
|
||||
option.WithTypeName(GetTypeFQN(User{})),
|
||||
option.WithIndexBy("UID"),
|
||||
option.WithDataURL(cfg.Repo.CS3.DataURL),
|
||||
option.WithDataPrefix(cfg.Repo.CS3.DataPrefix),
|
||||
option.WithJWTSecret(cfg.Repo.CS3.JWTSecret),
|
||||
option.WithProviderAddr(cfg.Repo.CS3.ProviderAddr),
|
||||
)
|
||||
|
||||
err := sut.Init()
|
||||
assert.NoError(b, err)
|
||||
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := sut.Add("abcdefg-123", "ignored")
|
||||
if err != nil {
|
||||
b.Error(err)
|
||||
}
|
||||
assert.NoError(b, err)
|
||||
}
|
||||
|
||||
_ = os.RemoveAll(dataDir)
|
||||
}
|
||||
|
||||
@@ -11,26 +11,28 @@ import (
|
||||
// User is a user.
|
||||
type User struct {
|
||||
ID, UserName, Email string
|
||||
UID int
|
||||
}
|
||||
|
||||
// Pet is a pet.
|
||||
type Pet struct {
|
||||
ID, Kind, Color, Name string
|
||||
UID int
|
||||
}
|
||||
|
||||
// Data mock data.
|
||||
var Data = map[string][]interface{}{
|
||||
"users": {
|
||||
User{ID: "abcdefg-123", UserName: "mikey", Email: "mikey@example.com"},
|
||||
User{ID: "hijklmn-456", UserName: "frank", Email: "frank@example.com"},
|
||||
User{ID: "ewf4ofk-555", UserName: "jacky", Email: "jacky@example.com"},
|
||||
User{ID: "rulan54-777", UserName: "jones", Email: "jones@example.com"},
|
||||
User{ID: "abcdefg-123", UserName: "mikey", Email: "mikey@example.com", UID: -1},
|
||||
User{ID: "hijklmn-456", UserName: "frank", Email: "frank@example.com", UID: -1},
|
||||
User{ID: "ewf4ofk-555", UserName: "jacky", Email: "jacky@example.com", UID: -1},
|
||||
User{ID: "rulan54-777", UserName: "jones", Email: "jones@example.com", UID: -1},
|
||||
},
|
||||
"pets": {
|
||||
Pet{ID: "rebef-123", Kind: "Dog", Color: "Brown", Name: "Waldo"},
|
||||
Pet{ID: "wefwe-456", Kind: "Cat", Color: "White", Name: "Snowy"},
|
||||
Pet{ID: "goefe-789", Kind: "Hog", Color: "Green", Name: "Dicky"},
|
||||
Pet{ID: "xadaf-189", Kind: "Hog", Color: "Green", Name: "Ricky"},
|
||||
Pet{ID: "rebef-123", Kind: "Dog", Color: "Brown", Name: "Waldo", UID: -1},
|
||||
Pet{ID: "wefwe-456", Kind: "Cat", Color: "White", Name: "Snowy", UID: -1},
|
||||
Pet{ID: "goefe-789", Kind: "Hog", Color: "Green", Name: "Dicky", UID: -1},
|
||||
Pet{ID: "xadaf-189", Kind: "Hog", Color: "Green", Name: "Ricky", UID: -1},
|
||||
},
|
||||
}
|
||||
|
||||
@@ -83,3 +85,28 @@ func WriteIndexTestDataCS3(t *testing.T, m map[string][]interface{}, pk string)
|
||||
|
||||
return rootDir
|
||||
}
|
||||
|
||||
// WriteIndexBenchmarkDataCS3 writes more data to disk.
|
||||
func WriteIndexBenchmarkDataCS3(b *testing.B, m map[string][]interface{}, pk string) string {
|
||||
rootDir := "/var/tmp/ocis/storage/users/data"
|
||||
for dirName := range m {
|
||||
fileTypePath := path.Join(rootDir, dirName)
|
||||
|
||||
if err := os.MkdirAll(fileTypePath, 0777); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
for _, u := range m[dirName] {
|
||||
data, err := json.Marshal(u)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
|
||||
pkVal := ValueOf(u, pk)
|
||||
if err := ioutil.WriteFile(path.Join(fileTypePath, pkVal), data, 0777); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return rootDir
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user