mirror/opencloud
1
0
Fork 0
mirror of https://github.com/opencloud-eu/opencloud.git synced 2026-01-04 11:19:39 -06:00
Code Issues Packages Projects Releases Wiki Activity

fix metadata for spaces registry

Browse Source 
Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de>
This commit is contained in:
Jörn Friedrich Dreyer
2021-12-07 14:48:56 +00:00
parent b527702572
commit a3702f722d
8 changed files with 93 additions and 118 deletions
Download Patch File Download Diff File Expand all files Collapse all files

67
accounts/pkg/storage/cs3.go
View File

@@ -6,18 +6,15 @@ import (
"path"
"path/filepath"
"github.com/cs3org/reva/pkg/auth/scope"
"github.com/cs3org/reva/pkg/errtypes"
"github.com/cs3org/reva/pkg/utils"
user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
v1beta11 "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
typesv1beta1 "github.com/cs3org/go-cs3apis/cs3/types/v1beta1"
"github.com/cs3org/reva/pkg/auth/scope"
revactx "github.com/cs3org/reva/pkg/ctx"
"github.com/cs3org/reva/pkg/rgrpc/todo/pool"
"github.com/cs3org/reva/pkg/token"
"github.com/cs3org/reva/pkg/token/manager/jwt"
"github.com/cs3org/reva/pkg/utils"
"github.com/owncloud/ocis/accounts/pkg/config"
"github.com/owncloud/ocis/accounts/pkg/proto/v0"
olog "github.com/owncloud/ocis/ocis-pkg/log"
@@ -25,16 +22,12 @@ import (
"google.golang.org/grpc/metadata"
)
const (
storageMountPath = "/meta"
)
// CS3Repo provides a cs3 implementation of the Repo interface
type CS3Repo struct {
cfg *config.Config
tm token.Manager
storageProvider provider.ProviderAPIClient
metadataStorage metadatastorage.MetadataStorage
metadataStorage *metadatastorage.MetadataStorage
}
// NewCS3Repo creates a new cs3 repo
@@ -57,60 +50,22 @@ func NewCS3Repo(cfg *config.Config) (Repo, error) {
return nil, err
}
repo := CS3Repo{
r := CS3Repo{
cfg: cfg,
tm: tokenManager,
storageProvider: client,
metadataStorage: ms,
metadataStorage: &ms,
}
if err := repo.Init(); err != nil {
ctx, err := r.getAuthenticatedContext(context.Background())
if err != nil {
return nil, err
}
if err := ms.Init(ctx, cfg.ServiceUser); err != nil {
return nil, err
}
return &repo, nil
}
// init creates the metadata space
func (r *CS3Repo) Init() (err error) {
ctx := context.Background()
ctx, err = r.getAuthenticatedContext(ctx)
if err != nil {
return err
}
// FIXME change CS3 api to allow sending a space id
cssr, err := r.storageProvider.CreateStorageSpace(ctx, &provider.CreateStorageSpaceRequest{
Opaque: &typesv1beta1.Opaque{
Map: map[string]*typesv1beta1.OpaqueEntry{
"spaceid": {
Decoder: "plain",
Value: []byte(r.cfg.ServiceUser.UUID),
},
},
},
Owner: &user.User{
Id: &user.UserId{
OpaqueId: r.cfg.ServiceUser.UUID,
},
Groups: []string{},
UidNumber: r.cfg.ServiceUser.UID,
GidNumber: r.cfg.ServiceUser.GID,
},
Name: "Metadata",
Type: "metadata",
})
switch {
case err != nil:
return err
case cssr.Status.Code == v1beta11.Code_CODE_OK:
r.metadataStorage.SpaceRoot = cssr.StorageSpace.Root
case cssr.Status.Code == v1beta11.Code_CODE_ALREADY_EXISTS:
// TODO make CreateStorageSpace return existing space?
r.metadataStorage.SpaceRoot = &provider.ResourceId{StorageId: r.cfg.ServiceUser.UUID, OpaqueId: r.cfg.ServiceUser.UUID}
default:
return errtypes.NewErrtypeFromStatus(cssr.Status)
}
return nil
return r, nil
}
// WriteAccount writes an account via cs3 and modifies the provided account (e.g. with a generated id).

2
go.mod
View File

@@ -241,4 +241,4 @@ require (
stash.kopano.io/kgol/oidc-go v0.3.2 // indirect
)
replace github.com/cs3org/reva => github.com/cs3org/reva v1.16.1-0.20211206210805-7ac4043bf153
replace github.com/cs3org/reva => github.com/cs3org/reva v1.16.1-0.20211207113801-1fa1ac1d3349

4
go.sum
View File

@@ -299,8 +299,8 @@ github.com/crewjam/saml v0.4.5/go.mod h1:qCJQpUtZte9R1ZjUBcW8qtCNlinbO363ooNl02S
github.com/cs3org/cato v0.0.0-20200828125504-e418fc54dd5e/go.mod h1:XJEZ3/EQuI3BXTp/6DUzFr850vlxq11I6satRtz0YQ4=
github.com/cs3org/go-cs3apis v0.0.0-20211104090126-8e972dca8304 h1:e/nIPR518vyvrulo9goAZTtYD6gFfu/2/9MDe6mTGcw=
github.com/cs3org/go-cs3apis v0.0.0-20211104090126-8e972dca8304/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY=
github.com/cs3org/reva v1.16.1-0.20211206210805-7ac4043bf153 h1:rufN1AuhGA7C+5CZSMFEkesSkjvCotXCj78WJFkjD0M=
github.com/cs3org/reva v1.16.1-0.20211206210805-7ac4043bf153/go.mod h1:3n/zVKsKTCL10Mwn2Nhtvn50gP5mA+933lQ2IYNAJso=
github.com/cs3org/reva v1.16.1-0.20211207113801-1fa1ac1d3349 h1:nqswCeR3uzHjMAwHdCLaf27+8niLI+nS8/u749Xh0cE=
github.com/cs3org/reva v1.16.1-0.20211207113801-1fa1ac1d3349/go.mod h1:3n/zVKsKTCL10Mwn2Nhtvn50gP5mA+933lQ2IYNAJso=
github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8 h1:Z9lwXumT5ACSmJ7WGnFl+OMLLjpz5uR2fyz7dC255FI=
github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8/go.mod h1:4abs/jPXcmJzYoYGF91JF9Uq9s/KL5n1jvFDix8KcqY=
github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4=

13
ocis-pkg/indexer/index/cs3/autoincrement.go
View File

@@ -94,6 +94,14 @@ func (idx *Autoincrement) Init() error {
}
idx.metadataStorage = &m
ctx, err := idx.getAuthenticatedContext(context.Background())
if err != nil {
return err
}
if err := idx.metadataStorage.Init(ctx, idx.cs3conf.ServiceUser); err != nil {
return err
}
if err := idx.makeDirIfNotExists(idx.indexBaseDir); err != nil {
return err
}
@@ -298,7 +306,10 @@ func (idx *Autoincrement) makeDirIfNotExists(folder string) error {
if err != nil {
return err
}
return storage.MakeDirIfNotExist(ctx, idx.storageProvider, folder)
return storage.MakeDirIfNotExist(ctx, idx.storageProvider, &provider.ResourceId{
StorageId: idx.cs3conf.ServiceUser.UUID,
OpaqueId: idx.cs3conf.ServiceUser.UUID,
}, folder)
}
func (idx *Autoincrement) next() (int, error) {

47
ocis-pkg/indexer/index/cs3/data_provider_client.go
View File

@@ -1,47 +0,0 @@
package cs3
import (
"io"
"net/http"
"path/filepath"
"strings"
)
type dataProviderClient struct {
client http.Client
spaceid string
baseURL string
}
func (d dataProviderClient) put(url string, body io.Reader, token string) (*http.Response, error) {
req, err := http.NewRequest(http.MethodPut, singleJoiningSlash(d.baseURL, filepath.Join("spaces", d.spaceid+"!"+d.spaceid, url)), body)
if err != nil {
return nil, err
}
req.Header.Add("x-access-token", token)
return d.client.Do(req)
}
func (d dataProviderClient) get(url string, token string) (*http.Response, error) {
req, err := http.NewRequest(http.MethodGet, singleJoiningSlash(d.baseURL, filepath.Join("spaces", d.spaceid+"!"+d.spaceid, url)), nil)
if err != nil {
return nil, err
}
req.Header.Add("x-access-token", token)
return d.client.Do(req)
}
// TODO: this is copied from proxy. Find a better solution or move it to ocis-pkg
func singleJoiningSlash(a, b string) string {
aslash := strings.HasSuffix(a, "/")
bslash := strings.HasPrefix(b, "/")
switch {
case aslash && bslash:
return a + b[1:]
case !aslash && !bslash:
return a + "/" + b
}
return a + b
}

13
ocis-pkg/indexer/index/cs3/non_unique.go
View File

@@ -96,6 +96,14 @@ func (idx *NonUnique) Init() error {
}
idx.metadataStorage = &m
ctx, err := idx.getAuthenticatedContext(context.Background())
if err != nil {
return err
}
if err := idx.metadataStorage.Init(ctx, idx.cs3conf.ServiceUser); err != nil {
return err
}
if err := idx.makeDirIfNotExists(idx.indexBaseDir); err != nil {
return err
}
@@ -328,7 +336,10 @@ func (idx *NonUnique) makeDirIfNotExists(folder string) error {
if err != nil {
return err
}
return storage.MakeDirIfNotExist(ctx, idx.storageProvider, folder)
return storage.MakeDirIfNotExist(ctx, idx.storageProvider, &provider.ResourceId{
StorageId: idx.cs3conf.ServiceUser.UUID,
OpaqueId: idx.cs3conf.ServiceUser.UUID,
}, folder)
}
func (idx *NonUnique) createSymlink(oldname, newname string) error {

8
ocis-pkg/indexer/index/cs3/unique.go
View File

@@ -91,6 +91,14 @@ func (idx *Unique) Init() error {
}
idx.metadataStorage = &m
ctx, err := idx.getAuthenticatedContext(context.Background())
if err != nil {
return err
}
if err := idx.metadataStorage.Init(ctx, idx.cs3conf.ServiceUser); err != nil {
return err
}
if err := idx.makeDirIfNotExists(idx.indexBaseDir); err != nil {
return err
}

57
ocis-pkg/metadata_storage/metadata_storage.go
View File

@@ -6,20 +6,20 @@ import (
"errors"
"io/ioutil"
"net/http"
"path"
user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
v1beta11 "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
typesv1beta1 "github.com/cs3org/go-cs3apis/cs3/types/v1beta1"
revactx "github.com/cs3org/reva/pkg/ctx"
"github.com/cs3org/reva/pkg/errtypes"
"github.com/cs3org/reva/pkg/rgrpc/todo/pool"
"github.com/cs3org/reva/pkg/utils"
"github.com/owncloud/ocis/accounts/pkg/config"
"google.golang.org/grpc/metadata"
)
const (
storageMountPath = "/meta"
)
func NewMetadataStorage(providerAddr string) (s MetadataStorage, err error) {
p, err := pool.GetStorageProviderServiceClient(providerAddr)
if err != nil {
return MetadataStorage{}, err
@@ -39,12 +39,49 @@ type MetadataStorage struct {
SpaceRoot *provider.ResourceId
}
// init creates the metadata space
func (ms *MetadataStorage) Init(ctx context.Context, serviceUser config.ServiceUser) (err error) {
// FIXME change CS3 api to allow sending a space id
cssr, err := ms.storageProvider.CreateStorageSpace(ctx, &provider.CreateStorageSpaceRequest{
Opaque: &typesv1beta1.Opaque{
Map: map[string]*typesv1beta1.OpaqueEntry{
"spaceid": {
Decoder: "plain",
Value: []byte(serviceUser.UUID),
},
},
},
Owner: &user.User{
Id: &user.UserId{
OpaqueId: serviceUser.UUID,
},
Groups: []string{},
UidNumber: serviceUser.UID,
GidNumber: serviceUser.GID,
},
Name: "Metadata",
Type: "metadata",
})
switch {
case err != nil:
return err
case cssr.Status.Code == v1beta11.Code_CODE_OK:
ms.SpaceRoot = cssr.StorageSpace.Root
case cssr.Status.Code == v1beta11.Code_CODE_ALREADY_EXISTS:
// TODO make CreateStorageSpace return existing space?
ms.SpaceRoot = &provider.ResourceId{StorageId: serviceUser.UUID, OpaqueId: serviceUser.UUID}
default:
return errtypes.NewErrtypeFromStatus(cssr.Status)
}
return nil
}
func (ms MetadataStorage) SimpleUpload(ctx context.Context, uploadpath string, content []byte) error {
ref := provider.InitiateFileUploadRequest{
Ref: &provider.Reference{
ResourceId: ms.SpaceRoot,
Path: path.Join(storageMountPath, uploadpath),
Path: utils.MakeRelativePath(uploadpath),
},
}
@@ -86,7 +123,7 @@ func (ms MetadataStorage) SimpleDownload(ctx context.Context, downloadpath strin
ref := provider.InitiateFileDownloadRequest{
Ref: &provider.Reference{
ResourceId: ms.SpaceRoot,
Path: path.Join(storageMountPath, downloadpath),
Path: utils.MakeRelativePath(downloadpath),
},
}
@@ -98,13 +135,13 @@ func (ms MetadataStorage) SimpleDownload(ctx context.Context, downloadpath strin
var endpoint string
for _, proto := range res.GetProtocols() {
if proto.Protocol == "simple" {
if proto.Protocol == "spaces" {
endpoint = proto.GetDownloadEndpoint()
break
}
}
if endpoint == "" {
return []byte{}, errors.New("metadata storage doesn't support the simple download protocol")
return []byte{}, errors.New("metadata storage doesn't support the spaces download protocol")
}
req, err := http.NewRequest(http.MethodGet, endpoint, nil)