From a3702f722da27e8a81526e0ac01f244ddfa6a0e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Tue, 7 Dec 2021 14:48:56 +0000 Subject: [PATCH] fix metadata for spaces registry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- accounts/pkg/storage/cs3.go | 67 +++---------------- go.mod | 2 +- go.sum | 4 +- ocis-pkg/indexer/index/cs3/autoincrement.go | 13 +++- .../indexer/index/cs3/data_provider_client.go | 47 ------------- ocis-pkg/indexer/index/cs3/non_unique.go | 13 +++- ocis-pkg/indexer/index/cs3/unique.go | 8 +++ ocis-pkg/metadata_storage/metadata_storage.go | 57 +++++++++++++--- 8 files changed, 93 insertions(+), 118 deletions(-) delete mode 100644 ocis-pkg/indexer/index/cs3/data_provider_client.go diff --git a/accounts/pkg/storage/cs3.go b/accounts/pkg/storage/cs3.go index 7418b60827..c50f68be52 100644 --- a/accounts/pkg/storage/cs3.go +++ b/accounts/pkg/storage/cs3.go @@ -6,18 +6,15 @@ import ( "path" "path/filepath" - "github.com/cs3org/reva/pkg/auth/scope" - "github.com/cs3org/reva/pkg/errtypes" - "github.com/cs3org/reva/pkg/utils" - user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" v1beta11 "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - typesv1beta1 "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" + "github.com/cs3org/reva/pkg/auth/scope" revactx "github.com/cs3org/reva/pkg/ctx" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" "github.com/cs3org/reva/pkg/token" "github.com/cs3org/reva/pkg/token/manager/jwt" + "github.com/cs3org/reva/pkg/utils" "github.com/owncloud/ocis/accounts/pkg/config" "github.com/owncloud/ocis/accounts/pkg/proto/v0" olog "github.com/owncloud/ocis/ocis-pkg/log" @@ -25,16 +22,12 @@ import ( "google.golang.org/grpc/metadata" ) -const ( - storageMountPath = "/meta" -) - // CS3Repo provides a cs3 implementation of the Repo interface type CS3Repo struct { cfg *config.Config tm token.Manager storageProvider provider.ProviderAPIClient - metadataStorage metadatastorage.MetadataStorage + metadataStorage *metadatastorage.MetadataStorage } // NewCS3Repo creates a new cs3 repo @@ -57,60 +50,22 @@ func NewCS3Repo(cfg *config.Config) (Repo, error) { return nil, err } - repo := CS3Repo{ + r := CS3Repo{ cfg: cfg, tm: tokenManager, storageProvider: client, - metadataStorage: ms, + metadataStorage: &ms, } - if err := repo.Init(); err != nil { + ctx, err := r.getAuthenticatedContext(context.Background()) + if err != nil { + return nil, err + } + if err := ms.Init(ctx, cfg.ServiceUser); err != nil { return nil, err } - return &repo, nil -} - -// init creates the metadata space -func (r *CS3Repo) Init() (err error) { - ctx := context.Background() - ctx, err = r.getAuthenticatedContext(ctx) - if err != nil { - return err - } - // FIXME change CS3 api to allow sending a space id - cssr, err := r.storageProvider.CreateStorageSpace(ctx, &provider.CreateStorageSpaceRequest{ - Opaque: &typesv1beta1.Opaque{ - Map: map[string]*typesv1beta1.OpaqueEntry{ - "spaceid": { - Decoder: "plain", - Value: []byte(r.cfg.ServiceUser.UUID), - }, - }, - }, - Owner: &user.User{ - Id: &user.UserId{ - OpaqueId: r.cfg.ServiceUser.UUID, - }, - Groups: []string{}, - UidNumber: r.cfg.ServiceUser.UID, - GidNumber: r.cfg.ServiceUser.GID, - }, - Name: "Metadata", - Type: "metadata", - }) - switch { - case err != nil: - return err - case cssr.Status.Code == v1beta11.Code_CODE_OK: - r.metadataStorage.SpaceRoot = cssr.StorageSpace.Root - case cssr.Status.Code == v1beta11.Code_CODE_ALREADY_EXISTS: - // TODO make CreateStorageSpace return existing space? - r.metadataStorage.SpaceRoot = &provider.ResourceId{StorageId: r.cfg.ServiceUser.UUID, OpaqueId: r.cfg.ServiceUser.UUID} - default: - return errtypes.NewErrtypeFromStatus(cssr.Status) - } - return nil + return r, nil } // WriteAccount writes an account via cs3 and modifies the provided account (e.g. with a generated id). diff --git a/go.mod b/go.mod index 0d9c015705..2565df75e3 100644 --- a/go.mod +++ b/go.mod @@ -241,4 +241,4 @@ require ( stash.kopano.io/kgol/oidc-go v0.3.2 // indirect ) -replace github.com/cs3org/reva => github.com/cs3org/reva v1.16.1-0.20211206210805-7ac4043bf153 +replace github.com/cs3org/reva => github.com/cs3org/reva v1.16.1-0.20211207113801-1fa1ac1d3349 diff --git a/go.sum b/go.sum index 98bd8ab522..97bb56ab01 100644 --- a/go.sum +++ b/go.sum @@ -299,8 +299,8 @@ github.com/crewjam/saml v0.4.5/go.mod h1:qCJQpUtZte9R1ZjUBcW8qtCNlinbO363ooNl02S github.com/cs3org/cato v0.0.0-20200828125504-e418fc54dd5e/go.mod h1:XJEZ3/EQuI3BXTp/6DUzFr850vlxq11I6satRtz0YQ4= github.com/cs3org/go-cs3apis v0.0.0-20211104090126-8e972dca8304 h1:e/nIPR518vyvrulo9goAZTtYD6gFfu/2/9MDe6mTGcw= github.com/cs3org/go-cs3apis v0.0.0-20211104090126-8e972dca8304/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY= -github.com/cs3org/reva v1.16.1-0.20211206210805-7ac4043bf153 h1:rufN1AuhGA7C+5CZSMFEkesSkjvCotXCj78WJFkjD0M= -github.com/cs3org/reva v1.16.1-0.20211206210805-7ac4043bf153/go.mod h1:3n/zVKsKTCL10Mwn2Nhtvn50gP5mA+933lQ2IYNAJso= +github.com/cs3org/reva v1.16.1-0.20211207113801-1fa1ac1d3349 h1:nqswCeR3uzHjMAwHdCLaf27+8niLI+nS8/u749Xh0cE= +github.com/cs3org/reva v1.16.1-0.20211207113801-1fa1ac1d3349/go.mod h1:3n/zVKsKTCL10Mwn2Nhtvn50gP5mA+933lQ2IYNAJso= github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8 h1:Z9lwXumT5ACSmJ7WGnFl+OMLLjpz5uR2fyz7dC255FI= github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8/go.mod h1:4abs/jPXcmJzYoYGF91JF9Uq9s/KL5n1jvFDix8KcqY= github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= diff --git a/ocis-pkg/indexer/index/cs3/autoincrement.go b/ocis-pkg/indexer/index/cs3/autoincrement.go index 263a24896a..1c988e9768 100644 --- a/ocis-pkg/indexer/index/cs3/autoincrement.go +++ b/ocis-pkg/indexer/index/cs3/autoincrement.go @@ -94,6 +94,14 @@ func (idx *Autoincrement) Init() error { } idx.metadataStorage = &m + ctx, err := idx.getAuthenticatedContext(context.Background()) + if err != nil { + return err + } + if err := idx.metadataStorage.Init(ctx, idx.cs3conf.ServiceUser); err != nil { + return err + } + if err := idx.makeDirIfNotExists(idx.indexBaseDir); err != nil { return err } @@ -298,7 +306,10 @@ func (idx *Autoincrement) makeDirIfNotExists(folder string) error { if err != nil { return err } - return storage.MakeDirIfNotExist(ctx, idx.storageProvider, folder) + return storage.MakeDirIfNotExist(ctx, idx.storageProvider, &provider.ResourceId{ + StorageId: idx.cs3conf.ServiceUser.UUID, + OpaqueId: idx.cs3conf.ServiceUser.UUID, + }, folder) } func (idx *Autoincrement) next() (int, error) { diff --git a/ocis-pkg/indexer/index/cs3/data_provider_client.go b/ocis-pkg/indexer/index/cs3/data_provider_client.go deleted file mode 100644 index baec5ddcb9..0000000000 --- a/ocis-pkg/indexer/index/cs3/data_provider_client.go +++ /dev/null @@ -1,47 +0,0 @@ -package cs3 - -import ( - "io" - "net/http" - "path/filepath" - "strings" -) - -type dataProviderClient struct { - client http.Client - spaceid string - baseURL string -} - -func (d dataProviderClient) put(url string, body io.Reader, token string) (*http.Response, error) { - req, err := http.NewRequest(http.MethodPut, singleJoiningSlash(d.baseURL, filepath.Join("spaces", d.spaceid+"!"+d.spaceid, url)), body) - if err != nil { - return nil, err - } - - req.Header.Add("x-access-token", token) - return d.client.Do(req) -} - -func (d dataProviderClient) get(url string, token string) (*http.Response, error) { - req, err := http.NewRequest(http.MethodGet, singleJoiningSlash(d.baseURL, filepath.Join("spaces", d.spaceid+"!"+d.spaceid, url)), nil) - if err != nil { - return nil, err - } - - req.Header.Add("x-access-token", token) - return d.client.Do(req) -} - -// TODO: this is copied from proxy. Find a better solution or move it to ocis-pkg -func singleJoiningSlash(a, b string) string { - aslash := strings.HasSuffix(a, "/") - bslash := strings.HasPrefix(b, "/") - switch { - case aslash && bslash: - return a + b[1:] - case !aslash && !bslash: - return a + "/" + b - } - return a + b -} diff --git a/ocis-pkg/indexer/index/cs3/non_unique.go b/ocis-pkg/indexer/index/cs3/non_unique.go index a839d05a62..3e0f1a222d 100644 --- a/ocis-pkg/indexer/index/cs3/non_unique.go +++ b/ocis-pkg/indexer/index/cs3/non_unique.go @@ -96,6 +96,14 @@ func (idx *NonUnique) Init() error { } idx.metadataStorage = &m + ctx, err := idx.getAuthenticatedContext(context.Background()) + if err != nil { + return err + } + if err := idx.metadataStorage.Init(ctx, idx.cs3conf.ServiceUser); err != nil { + return err + } + if err := idx.makeDirIfNotExists(idx.indexBaseDir); err != nil { return err } @@ -328,7 +336,10 @@ func (idx *NonUnique) makeDirIfNotExists(folder string) error { if err != nil { return err } - return storage.MakeDirIfNotExist(ctx, idx.storageProvider, folder) + return storage.MakeDirIfNotExist(ctx, idx.storageProvider, &provider.ResourceId{ + StorageId: idx.cs3conf.ServiceUser.UUID, + OpaqueId: idx.cs3conf.ServiceUser.UUID, + }, folder) } func (idx *NonUnique) createSymlink(oldname, newname string) error { diff --git a/ocis-pkg/indexer/index/cs3/unique.go b/ocis-pkg/indexer/index/cs3/unique.go index 0ba6760dd9..8bde7778f8 100644 --- a/ocis-pkg/indexer/index/cs3/unique.go +++ b/ocis-pkg/indexer/index/cs3/unique.go @@ -91,6 +91,14 @@ func (idx *Unique) Init() error { } idx.metadataStorage = &m + ctx, err := idx.getAuthenticatedContext(context.Background()) + if err != nil { + return err + } + if err := idx.metadataStorage.Init(ctx, idx.cs3conf.ServiceUser); err != nil { + return err + } + if err := idx.makeDirIfNotExists(idx.indexBaseDir); err != nil { return err } diff --git a/ocis-pkg/metadata_storage/metadata_storage.go b/ocis-pkg/metadata_storage/metadata_storage.go index 5a7a5dd4c2..51c293ec67 100644 --- a/ocis-pkg/metadata_storage/metadata_storage.go +++ b/ocis-pkg/metadata_storage/metadata_storage.go @@ -6,20 +6,20 @@ import ( "errors" "io/ioutil" "net/http" - "path" + user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + v1beta11 "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + typesv1beta1 "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" revactx "github.com/cs3org/reva/pkg/ctx" + "github.com/cs3org/reva/pkg/errtypes" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" + "github.com/cs3org/reva/pkg/utils" + "github.com/owncloud/ocis/accounts/pkg/config" "google.golang.org/grpc/metadata" ) -const ( - storageMountPath = "/meta" -) - func NewMetadataStorage(providerAddr string) (s MetadataStorage, err error) { - p, err := pool.GetStorageProviderServiceClient(providerAddr) if err != nil { return MetadataStorage{}, err @@ -39,12 +39,49 @@ type MetadataStorage struct { SpaceRoot *provider.ResourceId } +// init creates the metadata space +func (ms *MetadataStorage) Init(ctx context.Context, serviceUser config.ServiceUser) (err error) { + // FIXME change CS3 api to allow sending a space id + cssr, err := ms.storageProvider.CreateStorageSpace(ctx, &provider.CreateStorageSpaceRequest{ + Opaque: &typesv1beta1.Opaque{ + Map: map[string]*typesv1beta1.OpaqueEntry{ + "spaceid": { + Decoder: "plain", + Value: []byte(serviceUser.UUID), + }, + }, + }, + Owner: &user.User{ + Id: &user.UserId{ + OpaqueId: serviceUser.UUID, + }, + Groups: []string{}, + UidNumber: serviceUser.UID, + GidNumber: serviceUser.GID, + }, + Name: "Metadata", + Type: "metadata", + }) + switch { + case err != nil: + return err + case cssr.Status.Code == v1beta11.Code_CODE_OK: + ms.SpaceRoot = cssr.StorageSpace.Root + case cssr.Status.Code == v1beta11.Code_CODE_ALREADY_EXISTS: + // TODO make CreateStorageSpace return existing space? + ms.SpaceRoot = &provider.ResourceId{StorageId: serviceUser.UUID, OpaqueId: serviceUser.UUID} + default: + return errtypes.NewErrtypeFromStatus(cssr.Status) + } + return nil +} + func (ms MetadataStorage) SimpleUpload(ctx context.Context, uploadpath string, content []byte) error { ref := provider.InitiateFileUploadRequest{ Ref: &provider.Reference{ ResourceId: ms.SpaceRoot, - Path: path.Join(storageMountPath, uploadpath), + Path: utils.MakeRelativePath(uploadpath), }, } @@ -86,7 +123,7 @@ func (ms MetadataStorage) SimpleDownload(ctx context.Context, downloadpath strin ref := provider.InitiateFileDownloadRequest{ Ref: &provider.Reference{ ResourceId: ms.SpaceRoot, - Path: path.Join(storageMountPath, downloadpath), + Path: utils.MakeRelativePath(downloadpath), }, } @@ -98,13 +135,13 @@ func (ms MetadataStorage) SimpleDownload(ctx context.Context, downloadpath strin var endpoint string for _, proto := range res.GetProtocols() { - if proto.Protocol == "simple" { + if proto.Protocol == "spaces" { endpoint = proto.GetDownloadEndpoint() break } } if endpoint == "" { - return []byte{}, errors.New("metadata storage doesn't support the simple download protocol") + return []byte{}, errors.New("metadata storage doesn't support the spaces download protocol") } req, err := http.NewRequest(http.MethodGet, endpoint, nil)