From c067da5496ff326cbdb58cbab87c49a65ebd1359 Mon Sep 17 00:00:00 2001 From: Willy Kloucek Date: Thu, 2 Dec 2021 13:35:12 +0100 Subject: [PATCH 01/14] update reva --- changelog/unreleased/update-reva.md | 1 + go.mod | 2 +- go.sum | 4 ++-- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/changelog/unreleased/update-reva.md b/changelog/unreleased/update-reva.md index 9f3c9af52f..810cb656a8 100644 --- a/changelog/unreleased/update-reva.md +++ b/changelog/unreleased/update-reva.md @@ -7,3 +7,4 @@ This update includes: https://github.com/owncloud/ocis/pull/2835 +https://github.com/owncloud/ocis/pull/2837 diff --git a/go.mod b/go.mod index 46d89acd84..80e768568f 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( github.com/blevesearch/bleve/v2 v2.2.2 github.com/coreos/go-oidc/v3 v3.1.0 github.com/cs3org/go-cs3apis v0.0.0-20211104090126-8e972dca8304 - github.com/cs3org/reva v1.16.1-0.20211129095857-79fb2995dab9 + github.com/cs3org/reva v1.16.1-0.20211202113203-ba042373b390 github.com/disintegration/imaging v1.6.2 github.com/glauth/glauth/v2 v2.0.0-20211021011345-ef3151c28733 github.com/go-chi/chi/v5 v5.0.7 diff --git a/go.sum b/go.sum index 467ed18052..e6017d5992 100644 --- a/go.sum +++ b/go.sum @@ -299,8 +299,8 @@ github.com/crewjam/saml v0.4.5/go.mod h1:qCJQpUtZte9R1ZjUBcW8qtCNlinbO363ooNl02S github.com/cs3org/cato v0.0.0-20200828125504-e418fc54dd5e/go.mod h1:XJEZ3/EQuI3BXTp/6DUzFr850vlxq11I6satRtz0YQ4= github.com/cs3org/go-cs3apis v0.0.0-20211104090126-8e972dca8304 h1:e/nIPR518vyvrulo9goAZTtYD6gFfu/2/9MDe6mTGcw= github.com/cs3org/go-cs3apis v0.0.0-20211104090126-8e972dca8304/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY= -github.com/cs3org/reva v1.16.1-0.20211129095857-79fb2995dab9 h1:fgnLZB/tZCH4Kl2+ZBOpRd/dZLVF7VCrSFaL1FvZgwE= -github.com/cs3org/reva v1.16.1-0.20211129095857-79fb2995dab9/go.mod h1:3n/zVKsKTCL10Mwn2Nhtvn50gP5mA+933lQ2IYNAJso= +github.com/cs3org/reva v1.16.1-0.20211202113203-ba042373b390 h1:Lj+bd9lxiE0TshjZVvjx6nkWVbzD/UdCld1zScJzs4s= +github.com/cs3org/reva v1.16.1-0.20211202113203-ba042373b390/go.mod h1:3n/zVKsKTCL10Mwn2Nhtvn50gP5mA+933lQ2IYNAJso= github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8 h1:Z9lwXumT5ACSmJ7WGnFl+OMLLjpz5uR2fyz7dC255FI= github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8/go.mod h1:4abs/jPXcmJzYoYGF91JF9Uq9s/KL5n1jvFDix8KcqY= github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= From 3fd070061d66ba58293011b561b86090551b4118 Mon Sep 17 00:00:00 2001 From: Willy Kloucek Date: Thu, 2 Dec 2021 18:52:49 +0100 Subject: [PATCH 02/14] WIP: make the accounts service use the upload workflow --- accounts/pkg/config/config.go | 4 - accounts/pkg/config/mappings.go | 8 - accounts/pkg/service/v0/service.go | 2 - accounts/pkg/storage/cs3.go | 206 ++++++++++-------- accounts/pkg/storage/cs3_test.go | 2 - accounts/pkg/storage/errors.go | 2 +- ocis-pkg/indexer/index/cs3/autoincrement.go | 78 ++----- ocis-pkg/indexer/index/cs3/config.go | 12 + .../indexer/index/cs3/data_provider_client.go | 45 ---- ocis-pkg/indexer/index/cs3/non_unique.go | 89 +++----- ocis-pkg/indexer/index/cs3/storage.go | 107 +++++++++ ocis-pkg/indexer/index/cs3/unique.go | 94 ++------ 12 files changed, 307 insertions(+), 342 deletions(-) create mode 100644 ocis-pkg/indexer/index/cs3/config.go delete mode 100644 ocis-pkg/indexer/index/cs3/data_provider_client.go create mode 100644 ocis-pkg/indexer/index/cs3/storage.go diff --git a/accounts/pkg/config/config.go b/accounts/pkg/config/config.go index 4ae123db68..21bcd3dc25 100644 --- a/accounts/pkg/config/config.go +++ b/accounts/pkg/config/config.go @@ -89,8 +89,6 @@ type Disk struct { // CS3 is the cs3 implementation of the storage. type CS3 struct { ProviderAddr string `ocisConfig:"provider_addr"` - DataURL string `ocisConfig:"data_url"` - DataPrefix string `ocisConfig:"data_prefix"` JWTSecret string `ocisConfig:"jwt_secret"` } @@ -185,8 +183,6 @@ func DefaultConfig() *Config { }, CS3: CS3{ ProviderAddr: "localhost:9215", - DataURL: "http://localhost:9216", - DataPrefix: "data", JWTSecret: "Pive-Fumkiu4", }, }, diff --git a/accounts/pkg/config/mappings.go b/accounts/pkg/config/mappings.go index 4789fc8d69..00b13818b4 100644 --- a/accounts/pkg/config/mappings.go +++ b/accounts/pkg/config/mappings.go @@ -100,14 +100,6 @@ func structMappings(cfg *Config) []shared.EnvBinding { EnvVars: []string{"ACCOUNTS_STORAGE_CS3_PROVIDER_ADDR"}, Destination: &cfg.Repo.CS3.ProviderAddr, }, - { - EnvVars: []string{"ACCOUNTS_STORAGE_CS3_DATA_URL"}, - Destination: &cfg.Repo.CS3.DataURL, - }, - { - EnvVars: []string{"ACCOUNTS_STORAGE_CS3_DATA_PREFIX"}, - Destination: &cfg.Repo.CS3.DataPrefix, - }, { EnvVars: []string{"OCIS_JWT_SECRET", "ACCOUNTS_STORAGE_CS3_JWT_SECRET"}, Destination: &cfg.Repo.CS3.JWTSecret, diff --git a/accounts/pkg/service/v0/service.go b/accounts/pkg/service/v0/service.go index e4a5feefbd..cfea0f2209 100644 --- a/accounts/pkg/service/v0/service.go +++ b/accounts/pkg/service/v0/service.go @@ -138,8 +138,6 @@ func configFromSvc(cfg *config.Config) (*idxcfg.Config, error) { Backend: cfg.Repo.Backend, CS3: idxcfg.CS3{ ProviderAddr: cfg.Repo.CS3.ProviderAddr, - DataURL: cfg.Repo.CS3.DataURL, - DataPrefix: cfg.Repo.CS3.DataPrefix, JWTSecret: cfg.Repo.CS3.JWTSecret, }, } diff --git a/accounts/pkg/storage/cs3.go b/accounts/pkg/storage/cs3.go index 76cfa9dba4..ebe79e86bd 100644 --- a/accounts/pkg/storage/cs3.go +++ b/accounts/pkg/storage/cs3.go @@ -4,12 +4,11 @@ import ( "bytes" "context" "encoding/json" - "io" + "errors" "io/ioutil" "net/http" "path" "path/filepath" - "strings" "github.com/cs3org/reva/pkg/auth/scope" @@ -28,10 +27,10 @@ import ( // CS3Repo provides a cs3 implementation of the Repo interface type CS3Repo struct { - cfg *config.Config - tm token.Manager - storageProvider provider.ProviderAPIClient - dataProvider dataProviderClient // Used to create and download data via http, bypassing reva upload protocol + cfg *config.Config + tm token.Manager + storageProvider provider.ProviderAPIClient + dataGatewayClient *http.Client } // NewCS3Repo creates a new cs3 repo @@ -50,14 +49,10 @@ func NewCS3Repo(cfg *config.Config) (Repo, error) { } return CS3Repo{ - cfg: cfg, - tm: tokenManager, - storageProvider: client, - dataProvider: dataProviderClient{ - client: http.Client{ - Transport: http.DefaultTransport, - }, - }, + cfg: cfg, + tm: tokenManager, + storageProvider: client, + dataGatewayClient: http.DefaultClient, }, nil } @@ -78,14 +73,9 @@ func (r CS3Repo) WriteAccount(ctx context.Context, a *proto.Account) (err error) return err } - resp, err := r.dataProvider.put(r.accountURL(a.Id), bytes.NewReader(by), t) - if err != nil { - return err - } - if err = resp.Body.Close(); err != nil { - return err - } - return nil + err = r.uploadHelper(ctx, r.accountURL(a.Id), by) + return err + } // LoadAccount loads an account via cs3 by id and writes it to the provided account @@ -94,8 +84,9 @@ func (r CS3Repo) LoadAccount(ctx context.Context, id string, a *proto.Account) ( if err != nil { return err } + ctx = metadata.AppendToOutgoingContext(ctx, revactx.TokenHeader, t) - return r.loadAccount(id, t, a) + return r.loadAccount(ctx, id, a) } // LoadAccounts loads all the accounts from the cs3 api @@ -118,7 +109,7 @@ func (r CS3Repo) LoadAccounts(ctx context.Context, a *[]*proto.Account) (err err log := olog.NewLogger(olog.Pretty(r.cfg.Log.Pretty), olog.Color(r.cfg.Log.Color), olog.Level(r.cfg.Log.Level)) for i := range res.Infos { acc := &proto.Account{} - err := r.loadAccount(filepath.Base(res.Infos[i].Path), t, acc) + err := r.loadAccount(ctx, filepath.Base(res.Infos[i].Path), acc) if err != nil { log.Err(err).Msg("could not load account") continue @@ -128,24 +119,16 @@ func (r CS3Repo) LoadAccounts(ctx context.Context, a *[]*proto.Account) (err err return nil } -func (r CS3Repo) loadAccount(id string, t string, a *proto.Account) error { - resp, err := r.dataProvider.get(r.accountURL(id), t) +func (r CS3Repo) loadAccount(ctx context.Context, id string, a *proto.Account) error { + account, err := r.downloadHelper(ctx, r.accountURL(id)) if err != nil { + switch err.(type) { + case notFoundErr: + return notFoundErr{"account", id} + } return err } - - if resp.StatusCode != http.StatusOK { - return ¬FoundErr{"account", id} - } - - b, err := ioutil.ReadAll(resp.Body) - if err != nil { - return err - } - if err = resp.Body.Close(); err != nil { - return err - } - return json.Unmarshal(b, &a) + return json.Unmarshal(account, &a) } // DeleteAccount deletes an account via cs3 by id @@ -192,14 +175,8 @@ func (r CS3Repo) WriteGroup(ctx context.Context, g *proto.Group) (err error) { return err } - resp, err := r.dataProvider.put(r.groupURL(g.Id), bytes.NewReader(by), t) - if err != nil { - return err - } - if err = resp.Body.Close(); err != nil { - return err - } - return nil + err = r.uploadHelper(ctx, r.groupURL(g.Id), by) + return err } // LoadGroup loads a group via cs3 by id and writes it to the provided group @@ -208,8 +185,9 @@ func (r CS3Repo) LoadGroup(ctx context.Context, id string, g *proto.Group) (err if err != nil { return err } + ctx = metadata.AppendToOutgoingContext(ctx, revactx.TokenHeader, t) - return r.loadGroup(id, t, g) + return r.loadGroup(ctx, id, g) } // LoadGroups loads all the groups from the cs3 api @@ -232,7 +210,7 @@ func (r CS3Repo) LoadGroups(ctx context.Context, g *[]*proto.Group) (err error) log := olog.NewLogger(olog.Pretty(r.cfg.Log.Pretty), olog.Color(r.cfg.Log.Color), olog.Level(r.cfg.Log.Level)) for i := range res.Infos { grp := &proto.Group{} - err := r.loadGroup(filepath.Base(res.Infos[i].Path), t, grp) + err := r.loadGroup(ctx, filepath.Base(res.Infos[i].Path), grp) if err != nil { log.Err(err).Msg("could not load account") continue @@ -242,24 +220,16 @@ func (r CS3Repo) LoadGroups(ctx context.Context, g *[]*proto.Group) (err error) return nil } -func (r CS3Repo) loadGroup(id string, t string, g *proto.Group) error { - resp, err := r.dataProvider.get(r.groupURL(id), t) +func (r CS3Repo) loadGroup(ctx context.Context, id string, g *proto.Group) error { + group, err := r.downloadHelper(ctx, r.groupURL(id)) if err != nil { + switch err.(type) { + case notFoundErr: + return notFoundErr{"group", id} + } return err } - - if resp.StatusCode == http.StatusNotFound { - return ¬FoundErr{"group", id} - } - - b, err := ioutil.ReadAll(resp.Body) - if err != nil { - return err - } - if err = resp.Body.Close(); err != nil { - return err - } - return json.Unmarshal(b, &g) + return json.Unmarshal(group, &g) } // DeleteGroup deletes a group via cs3 by id @@ -311,11 +281,11 @@ func AuthenticateCS3(ctx context.Context, su config.ServiceUser, tm token.Manage } func (r CS3Repo) accountURL(id string) string { - return singleJoiningSlash(r.cfg.Repo.CS3.DataURL, path.Join(r.cfg.Repo.CS3.DataPrefix, accountsFolder, id)) + return path.Join(accountsFolder, id) } func (r CS3Repo) groupURL(id string) string { - return singleJoiningSlash(r.cfg.Repo.CS3.DataURL, path.Join(r.cfg.Repo.CS3.DataPrefix, groupsFolder, id)) + return path.Join(groupsFolder, id) } func (r CS3Repo) makeRootDirIfNotExist(ctx context.Context, folder string) error { @@ -349,39 +319,91 @@ func MakeDirIfNotExist(ctx context.Context, sp provider.ProviderAPIClient, folde return nil } -// TODO: this is copied from proxy. Find a better solution or move it to ocis-pkg -func singleJoiningSlash(a, b string) string { - aslash := strings.HasSuffix(a, "/") - bslash := strings.HasPrefix(b, "/") - switch { - case aslash && bslash: - return a + b[1:] - case !aslash && !bslash: - return a + "/" + b +func (r CS3Repo) uploadHelper(ctx context.Context, path string, content []byte) error { + + ref := provider.InitiateFileUploadRequest{ + Ref: &provider.Reference{ + Path: path, + }, } - return a + b -} -type dataProviderClient struct { - client http.Client -} - -func (d dataProviderClient) put(url string, body io.Reader, token string) (*http.Response, error) { - req, err := http.NewRequest(http.MethodPut, url, body) + res, err := r.storageProvider.InitiateFileUpload(ctx, &ref) if err != nil { - return nil, err + return err } - req.Header.Add(revactx.TokenHeader, token) - return d.client.Do(req) -} + var endpoint string -func (d dataProviderClient) get(url string, token string) (*http.Response, error) { - req, err := http.NewRequest(http.MethodGet, url, nil) + for _, proto := range res.GetProtocols() { + if proto.Protocol == "simple" { + endpoint = proto.GetUploadEndpoint() + break + } + } + if endpoint == "" { + return errors.New("metadata storage doesn't support the simple upload protocol") + } + + req, err := http.NewRequest(http.MethodPut, endpoint, bytes.NewReader(content)) if err != nil { - return nil, err + return err + } + resp, err := r.dataGatewayClient.Do(req) + if err != nil { + return err + } + if err = resp.Body.Close(); err != nil { + return err + } + return nil +} + +func (r CS3Repo) downloadHelper(ctx context.Context, path string) (content []byte, err error) { + + ref := provider.InitiateFileDownloadRequest{ + Ref: &provider.Reference{ + Path: path, + }, } - req.Header.Add(revactx.TokenHeader, token) - return d.client.Do(req) + res, err := r.storageProvider.InitiateFileDownload(ctx, &ref) + if err != nil { + return []byte{}, err + } + + var endpoint string + + for _, proto := range res.GetProtocols() { + if proto.Protocol == "simple" { + endpoint = proto.GetDownloadEndpoint() + break + } + } + if endpoint == "" { + return []byte{}, errors.New("metadata storage doesn't support the simple download protocol") + } + + req, err := http.NewRequest(http.MethodGet, endpoint, nil) + if err != nil { + return []byte{}, err + } + resp, err := r.dataGatewayClient.Do(req) + if err != nil { + return []byte{}, err + } + + if resp.StatusCode != http.StatusOK { + return []byte{}, ¬FoundErr{} + } + + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return []byte{}, err + } + + if err = resp.Body.Close(); err != nil { + return []byte{}, err + } + + return b, nil } diff --git a/accounts/pkg/storage/cs3_test.go b/accounts/pkg/storage/cs3_test.go index d788dc7ac3..70ef0afb97 100644 --- a/accounts/pkg/storage/cs3_test.go +++ b/accounts/pkg/storage/cs3_test.go @@ -17,8 +17,6 @@ package storage // Repo: config.Repo{ // CS3: config.CS3{ // ProviderAddr: "0.0.0.0:9215", -// DataURL: "http://localhost:9216", -// DataPrefix: "data", // }, // }, //} diff --git a/accounts/pkg/storage/errors.go b/accounts/pkg/storage/errors.go index 3dfa306e56..fd0c955cca 100644 --- a/accounts/pkg/storage/errors.go +++ b/accounts/pkg/storage/errors.go @@ -8,7 +8,7 @@ type notFoundErr struct { typ, id string } -func (e *notFoundErr) Error() string { +func (e notFoundErr) Error() string { return fmt.Sprintf("%s with id %s not found", e.typ, e.id) } diff --git a/ocis-pkg/indexer/index/cs3/autoincrement.go b/ocis-pkg/indexer/index/cs3/autoincrement.go index 2a638b542e..f548a36f7b 100644 --- a/ocis-pkg/indexer/index/cs3/autoincrement.go +++ b/ocis-pkg/indexer/index/cs3/autoincrement.go @@ -2,8 +2,6 @@ package cs3 import ( "context" - "fmt" - "io/ioutil" "net/http" "os" "path" @@ -20,7 +18,6 @@ import ( provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" revactx "github.com/cs3org/reva/pkg/ctx" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" - "github.com/cs3org/reva/pkg/token" "github.com/cs3org/reva/pkg/token/manager/jwt" "google.golang.org/grpc/metadata" @@ -37,9 +34,7 @@ type Autoincrement struct { indexBaseDir string indexRootDir string - tokenManager token.Manager - storageProvider provider.ProviderAPIClient - dataProvider dataProviderClient // Used to create and download data via http, bypassing reva upload protocol + metadataStorage *metadataStorage cs3conf *Config bound *option.Bound @@ -65,17 +60,9 @@ func NewAutoincrementIndex(o ...option.Option) index.Index { indexRootDir: path.Join(path.Join(opts.DataDir, "index.cs3"), strings.Join([]string{"autoincrement", opts.TypeName, opts.IndexBy}, ".")), cs3conf: &Config{ ProviderAddr: opts.ProviderAddr, - DataURL: opts.DataURL, - DataPrefix: opts.DataPrefix, JWTSecret: opts.JWTSecret, ServiceUser: opts.ServiceUser, }, - dataProvider: dataProviderClient{ - baseURL: singleJoiningSlash(opts.DataURL, opts.DataPrefix), - client: http.Client{ - Transport: http.DefaultTransport, - }, - }, } return u @@ -91,25 +78,22 @@ func (idx *Autoincrement) Init() error { return err } - idx.tokenManager = tokenManager - client, err := pool.GetStorageProviderServiceClient(idx.cs3conf.ProviderAddr) if err != nil { return err } - idx.storageProvider = client + idx.metadataStorage = &metadataStorage{ + tokenManager: tokenManager, + storageProvider: client, + dataGatewayClient: http.DefaultClient, + } - ctx, err := idx.getAuthenticatedContext(context.Background()) - if err != nil { + if err := idx.makeDirIfNotExists(idx.indexBaseDir); err != nil { return err } - if err := idx.makeDirIfNotExists(ctx, idx.indexBaseDir); err != nil { - return err - } - - if err := idx.makeDirIfNotExists(ctx, idx.indexRootDir); err != nil { + if err := idx.makeDirIfNotExists(idx.indexRootDir); err != nil { return err } @@ -175,7 +159,7 @@ func (idx *Autoincrement) Remove(id string, v string) error { } deletePath := path.Join("/meta", idx.indexRootDir, v) - resp, err := idx.storageProvider.Delete(ctx, &provider.DeleteRequest{ + resp, err := idx.metadataStorage.storageProvider.Delete(ctx, &provider.DeleteRequest{ Ref: &provider.Reference{ Path: deletePath, }, @@ -213,7 +197,7 @@ func (idx *Autoincrement) Search(pattern string) ([]string, error) { return nil, err } - res, err := idx.storageProvider.ListContainer(ctx, &provider.ListContainerRequest{ + res, err := idx.metadataStorage.storageProvider.ListContainer(ctx, &provider.ListContainerRequest{ Ref: &provider.Reference{ Path: path.Join("/meta", idx.indexRootDir), }, @@ -263,7 +247,7 @@ func (idx *Autoincrement) FilesDir() string { } func (idx *Autoincrement) createSymlink(oldname, newname string) error { - t, err := idx.authenticate(context.TODO()) + ctx, err := idx.getAuthenticatedContext(context.Background()) if err != nil { return err } @@ -272,52 +256,38 @@ func (idx *Autoincrement) createSymlink(oldname, newname string) error { return os.ErrExist } - resp, err := idx.dataProvider.put(newname, strings.NewReader(oldname), t) + err = idx.metadataStorage.uploadHelper(ctx, newname, []byte(oldname)) if err != nil { return err } - if err = resp.Body.Close(); err != nil { - return err - } - return nil } func (idx *Autoincrement) resolveSymlink(name string) (string, error) { - t, err := idx.authenticate(context.TODO()) + ctx, err := idx.getAuthenticatedContext(context.Background()) if err != nil { return "", err } - resp, err := idx.dataProvider.get(name, t) + b, err := idx.metadataStorage.downloadHelper(ctx, name) if err != nil { return "", err } - if resp.StatusCode != http.StatusOK { - if resp.StatusCode == http.StatusNotFound { - return "", os.ErrNotExist - } - - return "", fmt.Errorf("could not resolve symlink %s, got status %v", name, resp.StatusCode) - } - - b, err := ioutil.ReadAll(resp.Body) - if err != nil { - return "", err - } - if err = resp.Body.Close(); err != nil { - return "", err - } return string(b), err } -func (idx *Autoincrement) makeDirIfNotExists(ctx context.Context, folder string) error { - return storage.MakeDirIfNotExist(ctx, idx.storageProvider, folder) +func (idx *Autoincrement) makeDirIfNotExists(folder string) error { + ctx, err := idx.getAuthenticatedContext(context.Background()) + if err != nil { + return err + } + + return storage.MakeDirIfNotExist(ctx, idx.metadataStorage.storageProvider, folder) } func (idx *Autoincrement) authenticate(ctx context.Context) (token string, err error) { - return storage.AuthenticateCS3(ctx, idx.cs3conf.ServiceUser, idx.tokenManager) + return storage.AuthenticateCS3(ctx, idx.cs3conf.ServiceUser, idx.metadataStorage.tokenManager) } func (idx *Autoincrement) next() (int, error) { @@ -326,7 +296,7 @@ func (idx *Autoincrement) next() (int, error) { return -1, err } - res, err := idx.storageProvider.ListContainer(ctx, &provider.ListContainerRequest{ + res, err := idx.metadataStorage.storageProvider.ListContainer(ctx, &provider.ListContainerRequest{ Ref: &provider.Reference{ Path: path.Join("/meta", idx.indexRootDir), }, @@ -375,5 +345,5 @@ func (idx *Autoincrement) Delete() error { return err } - return deleteIndexRoot(ctx, idx.storageProvider, idx.indexRootDir) + return deleteIndexRoot(ctx, idx.metadataStorage.storageProvider, idx.indexRootDir) } diff --git a/ocis-pkg/indexer/index/cs3/config.go b/ocis-pkg/indexer/index/cs3/config.go new file mode 100644 index 0000000000..430927d4c6 --- /dev/null +++ b/ocis-pkg/indexer/index/cs3/config.go @@ -0,0 +1,12 @@ +package cs3 + +import ( + acccfg "github.com/owncloud/ocis/accounts/pkg/config" +) + +// Config represents cs3conf. Should be deprecated in favor of config.Config. +type Config struct { + ProviderAddr string + JWTSecret string + ServiceUser acccfg.ServiceUser +} diff --git a/ocis-pkg/indexer/index/cs3/data_provider_client.go b/ocis-pkg/indexer/index/cs3/data_provider_client.go deleted file mode 100644 index 3a2d42f7ed..0000000000 --- a/ocis-pkg/indexer/index/cs3/data_provider_client.go +++ /dev/null @@ -1,45 +0,0 @@ -package cs3 - -import ( - "io" - "net/http" - "strings" -) - -type dataProviderClient struct { - client http.Client - baseURL string -} - -func (d dataProviderClient) put(url string, body io.Reader, token string) (*http.Response, error) { - req, err := http.NewRequest(http.MethodPut, singleJoiningSlash(d.baseURL, url), body) - if err != nil { - return nil, err - } - - req.Header.Add("x-access-token", token) - return d.client.Do(req) -} - -func (d dataProviderClient) get(url string, token string) (*http.Response, error) { - req, err := http.NewRequest(http.MethodGet, singleJoiningSlash(d.baseURL, url), nil) - if err != nil { - return nil, err - } - - req.Header.Add("x-access-token", token) - return d.client.Do(req) -} - -// TODO: this is copied from proxy. Find a better solution or move it to ocis-pkg -func singleJoiningSlash(a, b string) string { - aslash := strings.HasSuffix(a, "/") - bslash := strings.HasPrefix(b, "/") - switch { - case aslash && bslash: - return a + b[1:] - case !aslash && !bslash: - return a + "/" + b - } - return a + b -} diff --git a/ocis-pkg/indexer/index/cs3/non_unique.go b/ocis-pkg/indexer/index/cs3/non_unique.go index d7715d60c3..1d686b9693 100644 --- a/ocis-pkg/indexer/index/cs3/non_unique.go +++ b/ocis-pkg/indexer/index/cs3/non_unique.go @@ -2,8 +2,6 @@ package cs3 import ( "context" - "fmt" - "io/ioutil" "net/http" "os" "path" @@ -16,7 +14,6 @@ import ( provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" revactx "github.com/cs3org/reva/pkg/ctx" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" - "github.com/cs3org/reva/pkg/token" "github.com/cs3org/reva/pkg/token/manager/jwt" idxerrs "github.com/owncloud/ocis/ocis-pkg/indexer/errors" "github.com/owncloud/ocis/ocis-pkg/indexer/index" @@ -38,9 +35,7 @@ type NonUnique struct { indexBaseDir string indexRootDir string - tokenManager token.Manager - storageProvider provider.ProviderAPIClient - dataProvider dataProviderClient // Used to create and download data via http, bypassing reva upload protocol + metadataStorage *metadataStorage cs3conf *Config } @@ -69,17 +64,9 @@ func NewNonUniqueIndexWithOptions(o ...option.Option) index.Index { indexRootDir: path.Join(path.Join(opts.DataDir, "index.cs3"), strings.Join([]string{"non_unique", opts.TypeName, opts.IndexBy}, ".")), cs3conf: &Config{ ProviderAddr: opts.ProviderAddr, - DataURL: opts.DataURL, - DataPrefix: opts.DataPrefix, JWTSecret: opts.JWTSecret, ServiceUser: opts.ServiceUser, }, - dataProvider: dataProviderClient{ - baseURL: singleJoiningSlash(opts.DataURL, opts.DataPrefix), - client: http.Client{ - Transport: http.DefaultTransport, - }, - }, } } @@ -93,27 +80,22 @@ func (idx *NonUnique) Init() error { return err } - idx.tokenManager = tokenManager - client, err := pool.GetStorageProviderServiceClient(idx.cs3conf.ProviderAddr) if err != nil { return err } - idx.storageProvider = client - - ctx := context.Background() - tk, err := idx.authenticate(ctx) - if err != nil { - return err + idx.metadataStorage = &metadataStorage{ + tokenManager: tokenManager, + storageProvider: client, + dataGatewayClient: http.DefaultClient, } - ctx = metadata.AppendToOutgoingContext(ctx, revactx.TokenHeader, tk) - if err := idx.makeDirIfNotExists(ctx, idx.indexBaseDir); err != nil { + if err := idx.makeDirIfNotExists(idx.indexBaseDir); err != nil { return err } - if err := idx.makeDirIfNotExists(ctx, idx.indexRootDir); err != nil { + if err := idx.makeDirIfNotExists(idx.indexRootDir); err != nil { return err } @@ -131,7 +113,7 @@ func (idx *NonUnique) Lookup(v string) ([]string, error) { return nil, err } - res, err := idx.storageProvider.ListContainer(ctx, &provider.ListContainerRequest{ + res, err := idx.metadataStorage.storageProvider.ListContainer(ctx, &provider.ListContainerRequest{ Ref: &provider.Reference{ Path: path.Join("/meta", idx.indexRootDir, v), }, @@ -156,13 +138,9 @@ func (idx *NonUnique) Add(id, v string) (string, error) { if idx.caseInsensitive { v = strings.ToLower(v) } - ctx, err := idx.getAuthenticatedContext(context.Background()) - if err != nil { - return "", err - } newName := path.Join(idx.indexRootDir, v) - if err := idx.makeDirIfNotExists(ctx, newName); err != nil { + if err := idx.makeDirIfNotExists(newName); err != nil { return "", err } @@ -191,7 +169,7 @@ func (idx *NonUnique) Remove(id string, v string) error { } deletePath := path.Join("/meta", idx.indexRootDir, v, id) - resp, err := idx.storageProvider.Delete(ctx, &provider.DeleteRequest{ + resp, err := idx.metadataStorage.storageProvider.Delete(ctx, &provider.DeleteRequest{ Ref: &provider.Reference{ Path: deletePath, }, @@ -206,7 +184,7 @@ func (idx *NonUnique) Remove(id string, v string) error { } toStat := path.Join("/meta", idx.indexRootDir, v) - lcResp, err := idx.storageProvider.ListContainer(ctx, &provider.ListContainerRequest{ + lcResp, err := idx.metadataStorage.storageProvider.ListContainer(ctx, &provider.ListContainerRequest{ Ref: &provider.Reference{ Path: toStat, }, @@ -217,7 +195,7 @@ func (idx *NonUnique) Remove(id string, v string) error { if len(lcResp.Infos) == 0 { deletePath = path.Join("/meta", idx.indexRootDir, v) - _, err := idx.storageProvider.Delete(ctx, &provider.DeleteRequest{ + _, err := idx.metadataStorage.storageProvider.Delete(ctx, &provider.DeleteRequest{ Ref: &provider.Reference{ Path: deletePath, }, @@ -261,7 +239,7 @@ func (idx *NonUnique) Search(pattern string) ([]string, error) { foldersMatched := make([]string, 0) matches := make([]string, 0) - res, err := idx.storageProvider.ListContainer(ctx, &provider.ListContainerRequest{ + res, err := idx.metadataStorage.storageProvider.ListContainer(ctx, &provider.ListContainerRequest{ Ref: &provider.Reference{ Path: path.Join("/meta", idx.indexRootDir), }, @@ -282,7 +260,7 @@ func (idx *NonUnique) Search(pattern string) ([]string, error) { } for i := range foldersMatched { - res, _ := idx.storageProvider.ListContainer(ctx, &provider.ListContainerRequest{ + res, _ := idx.metadataStorage.storageProvider.ListContainer(ctx, &provider.ListContainerRequest{ Ref: &provider.Reference{ Path: foldersMatched[i], }, @@ -316,12 +294,16 @@ func (idx *NonUnique) FilesDir() string { return idx.filesDir } -func (idx *NonUnique) makeDirIfNotExists(ctx context.Context, folder string) error { - return storage.MakeDirIfNotExist(ctx, idx.storageProvider, folder) +func (idx *NonUnique) makeDirIfNotExists(folder string) error { + ctx, err := idx.getAuthenticatedContext(context.Background()) + if err != nil { + return err + } + return storage.MakeDirIfNotExist(ctx, idx.metadataStorage.storageProvider, folder) } func (idx *NonUnique) createSymlink(oldname, newname string) error { - t, err := idx.authenticate(context.TODO()) + ctx, err := idx.getAuthenticatedContext(context.Background()) if err != nil { return err } @@ -330,43 +312,24 @@ func (idx *NonUnique) createSymlink(oldname, newname string) error { return os.ErrExist } - resp, err := idx.dataProvider.put(newname, strings.NewReader(oldname), t) + err = idx.metadataStorage.uploadHelper(ctx, newname, []byte(oldname)) if err != nil { return err } - if err = resp.Body.Close(); err != nil { - return err - } - return nil } func (idx *NonUnique) resolveSymlink(name string) (string, error) { - t, err := idx.authenticate(context.TODO()) + ctx, err := idx.getAuthenticatedContext(context.Background()) if err != nil { return "", err } - resp, err := idx.dataProvider.get(name, t) + b, err := idx.metadataStorage.downloadHelper(ctx, name) if err != nil { return "", err } - if resp.StatusCode != http.StatusOK { - if resp.StatusCode == http.StatusNotFound { - return "", os.ErrNotExist - } - - return "", fmt.Errorf("could not resolve symlink %s, got status %v", name, resp.StatusCode) - } - - b, err := ioutil.ReadAll(resp.Body) - if err != nil { - return "", err - } - if err = resp.Body.Close(); err != nil { - return "", err - } return string(b), err } @@ -386,9 +349,9 @@ func (idx *NonUnique) Delete() error { return err } - return deleteIndexRoot(ctx, idx.storageProvider, idx.indexRootDir) + return deleteIndexRoot(ctx, idx.metadataStorage.storageProvider, idx.indexRootDir) } func (idx *NonUnique) authenticate(ctx context.Context) (token string, err error) { - return storage.AuthenticateCS3(ctx, idx.cs3conf.ServiceUser, idx.tokenManager) + return storage.AuthenticateCS3(ctx, idx.cs3conf.ServiceUser, idx.metadataStorage.tokenManager) } diff --git a/ocis-pkg/indexer/index/cs3/storage.go b/ocis-pkg/indexer/index/cs3/storage.go new file mode 100644 index 0000000000..3e8d0fcd96 --- /dev/null +++ b/ocis-pkg/indexer/index/cs3/storage.go @@ -0,0 +1,107 @@ +package cs3 + +import ( + "bytes" + "context" + "errors" + "io/ioutil" + "net/http" + + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/cs3org/reva/pkg/token" +) + +type metadataStorage struct { + tokenManager token.Manager + storageProvider provider.ProviderAPIClient + dataGatewayClient *http.Client +} + +func (r metadataStorage) uploadHelper(ctx context.Context, path string, content []byte) error { + + ref := provider.InitiateFileUploadRequest{ + Ref: &provider.Reference{ + Path: path, + }, + } + + res, err := r.storageProvider.InitiateFileUpload(ctx, &ref) + if err != nil { + return err + } + + var endpoint string + + for _, proto := range res.GetProtocols() { + if proto.Protocol == "simple" { + endpoint = proto.GetUploadEndpoint() + break + } + } + if endpoint == "" { + return errors.New("metadata storage doesn't support the simple upload protocol") + } + + req, err := http.NewRequest(http.MethodPut, endpoint, bytes.NewReader(content)) + if err != nil { + return err + } + resp, err := r.dataGatewayClient.Do(req) + if err != nil { + return err + } + if err = resp.Body.Close(); err != nil { + return err + } + return nil +} + +func (r metadataStorage) downloadHelper(ctx context.Context, path string) (content []byte, err error) { + + ref := provider.InitiateFileDownloadRequest{ + Ref: &provider.Reference{ + Path: path, + }, + } + + res, err := r.storageProvider.InitiateFileDownload(ctx, &ref) + if err != nil { + return []byte{}, err + } + + var endpoint string + + for _, proto := range res.GetProtocols() { + if proto.Protocol == "simple" { + endpoint = proto.GetDownloadEndpoint() + break + } + } + if endpoint == "" { + return []byte{}, errors.New("metadata storage doesn't support the simple download protocol") + } + + req, err := http.NewRequest(http.MethodGet, endpoint, nil) + if err != nil { + return []byte{}, err + } + resp, err := r.dataGatewayClient.Do(req) + if err != nil { + return []byte{}, err + } + + //if resp.StatusCode != http.StatusOK { + // return []byte{}, ¬FoundErr{} + //} + + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return []byte{}, err + } + + if err = resp.Body.Close(); err != nil { + return []byte{}, err + } + + return b, nil +} diff --git a/ocis-pkg/indexer/index/cs3/unique.go b/ocis-pkg/indexer/index/cs3/unique.go index e94aee401a..1b9891ac7d 100644 --- a/ocis-pkg/indexer/index/cs3/unique.go +++ b/ocis-pkg/indexer/index/cs3/unique.go @@ -2,8 +2,6 @@ package cs3 import ( "context" - "fmt" - "io/ioutil" "net/http" "os" "path" @@ -12,13 +10,10 @@ import ( "github.com/owncloud/ocis/accounts/pkg/storage" - acccfg "github.com/owncloud/ocis/accounts/pkg/config" - v1beta11 "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" revactx "github.com/cs3org/reva/pkg/ctx" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" - "github.com/cs3org/reva/pkg/token" "github.com/cs3org/reva/pkg/token/manager/jwt" idxerrs "github.com/owncloud/ocis/ocis-pkg/indexer/errors" "github.com/owncloud/ocis/ocis-pkg/indexer/index" @@ -36,22 +31,11 @@ type Unique struct { indexBaseDir string indexRootDir string - tokenManager token.Manager - storageProvider provider.ProviderAPIClient - dataProvider dataProviderClient // Used to create and download data via http, bypassing reva upload protocol + metadataStorage *metadataStorage cs3conf *Config } -// Config represents cs3conf. Should be deprecated in favor of config.Config. -type Config struct { - ProviderAddr string - DataURL string - DataPrefix string - JWTSecret string - ServiceUser acccfg.ServiceUser -} - func init() { registry.IndexConstructorRegistry["cs3"]["unique"] = NewUniqueIndexWithOptions } @@ -73,17 +57,9 @@ func NewUniqueIndexWithOptions(o ...option.Option) index.Index { indexRootDir: path.Join(path.Join(opts.DataDir, "index.cs3"), strings.Join([]string{"unique", opts.TypeName, opts.IndexBy}, ".")), cs3conf: &Config{ ProviderAddr: opts.ProviderAddr, - DataURL: opts.DataURL, - DataPrefix: opts.DataPrefix, JWTSecret: opts.JWTSecret, ServiceUser: opts.ServiceUser, }, - dataProvider: dataProviderClient{ - baseURL: singleJoiningSlash(opts.DataURL, opts.DataPrefix), - client: http.Client{ - Transport: http.DefaultTransport, - }, - }, } return u @@ -94,32 +70,26 @@ func (idx *Unique) Init() error { tokenManager, err := jwt.New(map[string]interface{}{ "secret": idx.cs3conf.JWTSecret, }) - if err != nil { return err } - idx.tokenManager = tokenManager - client, err := pool.GetStorageProviderServiceClient(idx.cs3conf.ProviderAddr) if err != nil { return err } - idx.storageProvider = client - - ctx := context.Background() - tk, err := idx.authenticate(ctx) - if err != nil { - return err + idx.metadataStorage = &metadataStorage{ + tokenManager: tokenManager, + storageProvider: client, + dataGatewayClient: http.DefaultClient, } - ctx = metadata.AppendToOutgoingContext(ctx, revactx.TokenHeader, tk) - if err := idx.makeDirIfNotExists(ctx, idx.indexBaseDir); err != nil { + if err := idx.makeDirIfNotExists(idx.indexBaseDir); err != nil { return err } - if err := idx.makeDirIfNotExists(ctx, idx.indexRootDir); err != nil { + if err := idx.makeDirIfNotExists(idx.indexRootDir); err != nil { return err } @@ -182,15 +152,13 @@ func (idx *Unique) Remove(id string, v string) error { return err } - ctx := context.Background() - t, err := idx.authenticate(ctx) + ctx, err := idx.getAuthenticatedContext(context.Background()) if err != nil { return err } deletePath := path.Join("/meta", idx.indexRootDir, v) - ctx = metadata.AppendToOutgoingContext(ctx, revactx.TokenHeader, t) - resp, err := idx.storageProvider.Delete(ctx, &provider.DeleteRequest{ + resp, err := idx.metadataStorage.storageProvider.Delete(ctx, &provider.DeleteRequest{ Ref: &provider.Reference{ Path: deletePath, }, @@ -232,14 +200,12 @@ func (idx *Unique) Search(pattern string) ([]string, error) { pattern = strings.ToLower(pattern) } - ctx := context.Background() - t, err := idx.authenticate(ctx) + ctx, err := idx.getAuthenticatedContext(context.Background()) if err != nil { return nil, err } - ctx = metadata.AppendToOutgoingContext(ctx, revactx.TokenHeader, t) - res, err := idx.storageProvider.ListContainer(ctx, &provider.ListContainerRequest{ + res, err := idx.metadataStorage.storageProvider.ListContainer(ctx, &provider.ListContainerRequest{ Ref: &provider.Reference{ Path: path.Join("/meta", idx.indexRootDir), }, @@ -289,7 +255,7 @@ func (idx *Unique) FilesDir() string { } func (idx *Unique) createSymlink(oldname, newname string) error { - t, err := idx.authenticate(context.TODO()) + ctx, err := idx.getAuthenticatedContext(context.Background()) if err != nil { return err } @@ -298,52 +264,38 @@ func (idx *Unique) createSymlink(oldname, newname string) error { return os.ErrExist } - resp, err := idx.dataProvider.put(newname, strings.NewReader(oldname), t) + err = idx.metadataStorage.uploadHelper(ctx, newname, []byte(oldname)) if err != nil { return err } - if err = resp.Body.Close(); err != nil { - return err - } return nil } func (idx *Unique) resolveSymlink(name string) (string, error) { - t, err := idx.authenticate(context.TODO()) + ctx, err := idx.getAuthenticatedContext(context.Background()) if err != nil { return "", err } - resp, err := idx.dataProvider.get(name, t) + b, err := idx.metadataStorage.downloadHelper(ctx, name) if err != nil { return "", err } - if resp.StatusCode != http.StatusOK { - if resp.StatusCode == http.StatusNotFound { - return "", os.ErrNotExist - } - - return "", fmt.Errorf("could not resolve symlink %s, got status %v", name, resp.StatusCode) - } - - b, err := ioutil.ReadAll(resp.Body) - if err != nil { - return "", err - } - if err = resp.Body.Close(); err != nil { - return "", err - } return string(b), err } -func (idx *Unique) makeDirIfNotExists(ctx context.Context, folder string) error { - return storage.MakeDirIfNotExist(ctx, idx.storageProvider, folder) +func (idx *Unique) makeDirIfNotExists(folder string) error { + ctx, err := idx.getAuthenticatedContext(context.Background()) + if err != nil { + return err + } + return storage.MakeDirIfNotExist(ctx, idx.metadataStorage.storageProvider, folder) } func (idx *Unique) authenticate(ctx context.Context) (token string, err error) { - return storage.AuthenticateCS3(ctx, idx.cs3conf.ServiceUser, idx.tokenManager) + return storage.AuthenticateCS3(ctx, idx.cs3conf.ServiceUser, idx.metadataStorage.tokenManager) } func (idx *Unique) getAuthenticatedContext(ctx context.Context) (context.Context, error) { @@ -362,5 +314,5 @@ func (idx *Unique) Delete() error { return err } - return deleteIndexRoot(ctx, idx.storageProvider, idx.indexRootDir) + return deleteIndexRoot(ctx, idx.metadataStorage.storageProvider, idx.indexRootDir) } From 08641a8e45d5af9aed7efe01a4976f92a5ee136c Mon Sep 17 00:00:00 2001 From: David Christofas Date: Fri, 3 Dec 2021 17:43:01 +0100 Subject: [PATCH 03/14] fix download helper for accounts service and indexer --- accounts/pkg/storage/cs3.go | 27 +++++++++++------ ocis-pkg/indexer/index/cs3/storage.go | 43 ++++++++++++++++++++++----- storage/pkg/config/config.go | 2 +- 3 files changed, 54 insertions(+), 18 deletions(-) diff --git a/accounts/pkg/storage/cs3.go b/accounts/pkg/storage/cs3.go index ebe79e86bd..8ff78241bf 100644 --- a/accounts/pkg/storage/cs3.go +++ b/accounts/pkg/storage/cs3.go @@ -25,6 +25,10 @@ import ( "google.golang.org/grpc/metadata" ) +const ( + storageMountPath = "/meta" +) + // CS3Repo provides a cs3 implementation of the Repo interface type CS3Repo struct { cfg *config.Config @@ -99,7 +103,7 @@ func (r CS3Repo) LoadAccounts(ctx context.Context, a *[]*proto.Account) (err err ctx = metadata.AppendToOutgoingContext(ctx, revactx.TokenHeader, t) res, err := r.storageProvider.ListContainer(ctx, &provider.ListContainerRequest{ Ref: &provider.Reference{ - Path: path.Join("/meta", accountsFolder), + Path: path.Join(storageMountPath, accountsFolder), }, }) if err != nil { @@ -142,7 +146,7 @@ func (r CS3Repo) DeleteAccount(ctx context.Context, id string) (err error) { resp, err := r.storageProvider.Delete(ctx, &provider.DeleteRequest{ Ref: &provider.Reference{ - Path: path.Join("/meta", accountsFolder, id), + Path: path.Join(storageMountPath, accountsFolder, id), }, }) @@ -200,7 +204,7 @@ func (r CS3Repo) LoadGroups(ctx context.Context, g *[]*proto.Group) (err error) ctx = metadata.AppendToOutgoingContext(ctx, revactx.TokenHeader, t) res, err := r.storageProvider.ListContainer(ctx, &provider.ListContainerRequest{ Ref: &provider.Reference{ - Path: path.Join("/meta", groupsFolder), + Path: path.Join(storageMountPath, groupsFolder), }, }) if err != nil { @@ -243,7 +247,7 @@ func (r CS3Repo) DeleteGroup(ctx context.Context, id string) (err error) { resp, err := r.storageProvider.Delete(ctx, &provider.DeleteRequest{ Ref: &provider.Reference{ - Path: path.Join("/meta", groupsFolder, id), + Path: path.Join(storageMountPath, groupsFolder, id), }, }) @@ -268,6 +272,7 @@ func AuthenticateCS3(ctx context.Context, su config.ServiceUser, tm token.Manage u := &user.User{ Id: &user.UserId{ OpaqueId: su.UUID, + Type: user.UserType_USER_TYPE_APPLICATION, }, Groups: []string{}, UidNumber: su.UID, @@ -295,7 +300,7 @@ func (r CS3Repo) makeRootDirIfNotExist(ctx context.Context, folder string) error // MakeDirIfNotExist will create a root node in the metadata storage. Requires an authenticated context. func MakeDirIfNotExist(ctx context.Context, sp provider.ProviderAPIClient, folder string) error { var rootPathRef = &provider.Reference{ - Path: path.Join("/meta", folder), + Path: path.Join(storageMountPath, folder), } resp, err := sp.Stat(ctx, &provider.StatRequest{ @@ -319,11 +324,11 @@ func MakeDirIfNotExist(ctx context.Context, sp provider.ProviderAPIClient, folde return nil } -func (r CS3Repo) uploadHelper(ctx context.Context, path string, content []byte) error { +func (r CS3Repo) uploadHelper(ctx context.Context, uploadpath string, content []byte) error { ref := provider.InitiateFileUploadRequest{ Ref: &provider.Reference{ - Path: path, + Path: path.Join(storageMountPath, uploadpath), }, } @@ -348,6 +353,8 @@ func (r CS3Repo) uploadHelper(ctx context.Context, path string, content []byte) if err != nil { return err } + md, _ := metadata.FromOutgoingContext(ctx) + req.Header.Add(revactx.TokenHeader, md.Get(revactx.TokenHeader)[0]) resp, err := r.dataGatewayClient.Do(req) if err != nil { return err @@ -358,11 +365,11 @@ func (r CS3Repo) uploadHelper(ctx context.Context, path string, content []byte) return nil } -func (r CS3Repo) downloadHelper(ctx context.Context, path string) (content []byte, err error) { +func (r CS3Repo) downloadHelper(ctx context.Context, downloadpath string) (content []byte, err error) { ref := provider.InitiateFileDownloadRequest{ Ref: &provider.Reference{ - Path: path, + Path: path.Join(storageMountPath, downloadpath), }, } @@ -387,6 +394,8 @@ func (r CS3Repo) downloadHelper(ctx context.Context, path string) (content []byt if err != nil { return []byte{}, err } + md, _ := metadata.FromOutgoingContext(ctx) + req.Header.Add(revactx.TokenHeader, md.Get(revactx.TokenHeader)[0]) resp, err := r.dataGatewayClient.Do(req) if err != nil { return []byte{}, err diff --git a/ocis-pkg/indexer/index/cs3/storage.go b/ocis-pkg/indexer/index/cs3/storage.go index 3e8d0fcd96..012e2c023c 100644 --- a/ocis-pkg/indexer/index/cs3/storage.go +++ b/ocis-pkg/indexer/index/cs3/storage.go @@ -4,11 +4,19 @@ import ( "bytes" "context" "errors" + "fmt" "io/ioutil" "net/http" + "path" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + revactx "github.com/cs3org/reva/pkg/ctx" "github.com/cs3org/reva/pkg/token" + "google.golang.org/grpc/metadata" +) + +const ( + storageMountPath = "/meta" ) type metadataStorage struct { @@ -17,11 +25,11 @@ type metadataStorage struct { dataGatewayClient *http.Client } -func (r metadataStorage) uploadHelper(ctx context.Context, path string, content []byte) error { +func (r metadataStorage) uploadHelper(ctx context.Context, uploadpath string, content []byte) error { ref := provider.InitiateFileUploadRequest{ Ref: &provider.Reference{ - Path: path, + Path: path.Join(storageMountPath, uploadpath), }, } @@ -46,6 +54,9 @@ func (r metadataStorage) uploadHelper(ctx context.Context, path string, content if err != nil { return err } + + md, _ := metadata.FromOutgoingContext(ctx) + req.Header.Add(revactx.TokenHeader, md.Get(revactx.TokenHeader)[0]) resp, err := r.dataGatewayClient.Do(req) if err != nil { return err @@ -56,11 +67,10 @@ func (r metadataStorage) uploadHelper(ctx context.Context, path string, content return nil } -func (r metadataStorage) downloadHelper(ctx context.Context, path string) (content []byte, err error) { - +func (r metadataStorage) downloadHelper(ctx context.Context, downloadpath string) (content []byte, err error) { ref := provider.InitiateFileDownloadRequest{ Ref: &provider.Reference{ - Path: path, + Path: path.Join(storageMountPath, downloadpath), }, } @@ -85,14 +95,17 @@ func (r metadataStorage) downloadHelper(ctx context.Context, path string) (conte if err != nil { return []byte{}, err } + + md, _ := metadata.FromOutgoingContext(ctx) + req.Header.Add(revactx.TokenHeader, md.Get(revactx.TokenHeader)[0]) resp, err := r.dataGatewayClient.Do(req) if err != nil { return []byte{}, err } - //if resp.StatusCode != http.StatusOK { - // return []byte{}, ¬FoundErr{} - //} + if resp.StatusCode != http.StatusOK { + return []byte{}, ¬FoundErr{} + } b, err := ioutil.ReadAll(resp.Body) if err != nil { @@ -105,3 +118,17 @@ func (r metadataStorage) downloadHelper(ctx context.Context, path string) (conte return b, nil } + +type notFoundErr struct { + typ, id string +} + +func (e notFoundErr) Error() string { + return fmt.Sprintf("%s with id %s not found", e.typ, e.id) +} + +// IsNotFoundErr can be returned by repo Load and Delete operations +func IsNotFoundErr(e error) bool { + _, ok := e.(*notFoundErr) + return ok +} diff --git a/storage/pkg/config/config.go b/storage/pkg/config/config.go index c1a201ac6c..0687716b31 100644 --- a/storage/pkg/config/config.go +++ b/storage/pkg/config/config.go @@ -922,7 +922,7 @@ func DefaultConfig() *Config { }, Driver: "ocis", ExposeDataServer: false, - DataServerURL: "http://localhost:9216", + DataServerURL: "http://localhost:9216/data", TempFolder: path.Join(defaults.BaseDataPath(), "tmp", "metadata"), DataProvider: DataProvider{}, }, From d961decd64ade54d2cd65aa293825406933c578a Mon Sep 17 00:00:00 2001 From: Willy Kloucek Date: Mon, 6 Dec 2021 08:53:18 +0100 Subject: [PATCH 04/14] update reva --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 80e768568f..4d989478e7 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( github.com/blevesearch/bleve/v2 v2.2.2 github.com/coreos/go-oidc/v3 v3.1.0 github.com/cs3org/go-cs3apis v0.0.0-20211104090126-8e972dca8304 - github.com/cs3org/reva v1.16.1-0.20211202113203-ba042373b390 + github.com/cs3org/reva v1.16.1-0.20211203225713-939768a1af06 github.com/disintegration/imaging v1.6.2 github.com/glauth/glauth/v2 v2.0.0-20211021011345-ef3151c28733 github.com/go-chi/chi/v5 v5.0.7 diff --git a/go.sum b/go.sum index e6017d5992..e9bd5fdb02 100644 --- a/go.sum +++ b/go.sum @@ -301,6 +301,8 @@ github.com/cs3org/go-cs3apis v0.0.0-20211104090126-8e972dca8304 h1:e/nIPR518vyvr github.com/cs3org/go-cs3apis v0.0.0-20211104090126-8e972dca8304/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY= github.com/cs3org/reva v1.16.1-0.20211202113203-ba042373b390 h1:Lj+bd9lxiE0TshjZVvjx6nkWVbzD/UdCld1zScJzs4s= github.com/cs3org/reva v1.16.1-0.20211202113203-ba042373b390/go.mod h1:3n/zVKsKTCL10Mwn2Nhtvn50gP5mA+933lQ2IYNAJso= +github.com/cs3org/reva v1.16.1-0.20211203225713-939768a1af06 h1:NP+Zvli7+9USaDss/+Ywk4KJ0H7n82UHZiU4V+x25I0= +github.com/cs3org/reva v1.16.1-0.20211203225713-939768a1af06/go.mod h1:3n/zVKsKTCL10Mwn2Nhtvn50gP5mA+933lQ2IYNAJso= github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8 h1:Z9lwXumT5ACSmJ7WGnFl+OMLLjpz5uR2fyz7dC255FI= github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8/go.mod h1:4abs/jPXcmJzYoYGF91JF9Uq9s/KL5n1jvFDix8KcqY= github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= From 19a51a4402ff915835ec5f4242e99cef45b581a8 Mon Sep 17 00:00:00 2001 From: Willy Kloucek Date: Mon, 6 Dec 2021 09:16:00 +0100 Subject: [PATCH 05/14] fix not found error --- go.sum | 4 ---- ocis-pkg/indexer/index/cs3/autoincrement.go | 4 ++++ ocis-pkg/indexer/index/cs3/non_unique.go | 4 ++++ ocis-pkg/indexer/index/cs3/unique.go | 4 ++++ 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/go.sum b/go.sum index e9bd5fdb02..ac4436d91d 100644 --- a/go.sum +++ b/go.sum @@ -299,10 +299,6 @@ github.com/crewjam/saml v0.4.5/go.mod h1:qCJQpUtZte9R1ZjUBcW8qtCNlinbO363ooNl02S github.com/cs3org/cato v0.0.0-20200828125504-e418fc54dd5e/go.mod h1:XJEZ3/EQuI3BXTp/6DUzFr850vlxq11I6satRtz0YQ4= github.com/cs3org/go-cs3apis v0.0.0-20211104090126-8e972dca8304 h1:e/nIPR518vyvrulo9goAZTtYD6gFfu/2/9MDe6mTGcw= github.com/cs3org/go-cs3apis v0.0.0-20211104090126-8e972dca8304/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY= -github.com/cs3org/reva v1.16.1-0.20211202113203-ba042373b390 h1:Lj+bd9lxiE0TshjZVvjx6nkWVbzD/UdCld1zScJzs4s= -github.com/cs3org/reva v1.16.1-0.20211202113203-ba042373b390/go.mod h1:3n/zVKsKTCL10Mwn2Nhtvn50gP5mA+933lQ2IYNAJso= -github.com/cs3org/reva v1.16.1-0.20211203225713-939768a1af06 h1:NP+Zvli7+9USaDss/+Ywk4KJ0H7n82UHZiU4V+x25I0= -github.com/cs3org/reva v1.16.1-0.20211203225713-939768a1af06/go.mod h1:3n/zVKsKTCL10Mwn2Nhtvn50gP5mA+933lQ2IYNAJso= github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8 h1:Z9lwXumT5ACSmJ7WGnFl+OMLLjpz5uR2fyz7dC255FI= github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8/go.mod h1:4abs/jPXcmJzYoYGF91JF9Uq9s/KL5n1jvFDix8KcqY= github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= diff --git a/ocis-pkg/indexer/index/cs3/autoincrement.go b/ocis-pkg/indexer/index/cs3/autoincrement.go index f548a36f7b..cd88396ea2 100644 --- a/ocis-pkg/indexer/index/cs3/autoincrement.go +++ b/ocis-pkg/indexer/index/cs3/autoincrement.go @@ -271,6 +271,10 @@ func (idx *Autoincrement) resolveSymlink(name string) (string, error) { b, err := idx.metadataStorage.downloadHelper(ctx, name) if err != nil { + switch err.(type) { + case notFoundErr: + return "", os.ErrNotExist + } return "", err } diff --git a/ocis-pkg/indexer/index/cs3/non_unique.go b/ocis-pkg/indexer/index/cs3/non_unique.go index 1d686b9693..9492886819 100644 --- a/ocis-pkg/indexer/index/cs3/non_unique.go +++ b/ocis-pkg/indexer/index/cs3/non_unique.go @@ -327,6 +327,10 @@ func (idx *NonUnique) resolveSymlink(name string) (string, error) { b, err := idx.metadataStorage.downloadHelper(ctx, name) if err != nil { + switch err.(type) { + case notFoundErr: + return "", os.ErrNotExist + } return "", err } diff --git a/ocis-pkg/indexer/index/cs3/unique.go b/ocis-pkg/indexer/index/cs3/unique.go index 1b9891ac7d..cd9b56615d 100644 --- a/ocis-pkg/indexer/index/cs3/unique.go +++ b/ocis-pkg/indexer/index/cs3/unique.go @@ -280,6 +280,10 @@ func (idx *Unique) resolveSymlink(name string) (string, error) { b, err := idx.metadataStorage.downloadHelper(ctx, name) if err != nil { + switch err.(type) { + case notFoundErr: + return "", os.ErrNotExist + } return "", err } From c37800eb7203fa361df65603cd420779d0dec579 Mon Sep 17 00:00:00 2001 From: Willy Kloucek Date: Mon, 6 Dec 2021 10:15:18 +0100 Subject: [PATCH 06/14] fix user creation --- accounts/pkg/storage/cs3.go | 10 ++++------ go.sum | 2 ++ ocis-pkg/indexer/index/cs3/autoincrement.go | 3 +-- ocis-pkg/indexer/index/cs3/non_unique.go | 3 +-- ocis-pkg/indexer/index/cs3/unique.go | 3 +-- 5 files changed, 9 insertions(+), 12 deletions(-) diff --git a/accounts/pkg/storage/cs3.go b/accounts/pkg/storage/cs3.go index 8ff78241bf..d4d4321d08 100644 --- a/accounts/pkg/storage/cs3.go +++ b/accounts/pkg/storage/cs3.go @@ -126,9 +126,8 @@ func (r CS3Repo) LoadAccounts(ctx context.Context, a *[]*proto.Account) (err err func (r CS3Repo) loadAccount(ctx context.Context, id string, a *proto.Account) error { account, err := r.downloadHelper(ctx, r.accountURL(id)) if err != nil { - switch err.(type) { - case notFoundErr: - return notFoundErr{"account", id} + if IsNotFoundErr(err) { + return ¬FoundErr{"account", id} } return err } @@ -227,9 +226,8 @@ func (r CS3Repo) LoadGroups(ctx context.Context, g *[]*proto.Group) (err error) func (r CS3Repo) loadGroup(ctx context.Context, id string, g *proto.Group) error { group, err := r.downloadHelper(ctx, r.groupURL(id)) if err != nil { - switch err.(type) { - case notFoundErr: - return notFoundErr{"group", id} + if IsNotFoundErr(err) { + return ¬FoundErr{"group", id} } return err } diff --git a/go.sum b/go.sum index ac4436d91d..28a54a77ad 100644 --- a/go.sum +++ b/go.sum @@ -299,6 +299,8 @@ github.com/crewjam/saml v0.4.5/go.mod h1:qCJQpUtZte9R1ZjUBcW8qtCNlinbO363ooNl02S github.com/cs3org/cato v0.0.0-20200828125504-e418fc54dd5e/go.mod h1:XJEZ3/EQuI3BXTp/6DUzFr850vlxq11I6satRtz0YQ4= github.com/cs3org/go-cs3apis v0.0.0-20211104090126-8e972dca8304 h1:e/nIPR518vyvrulo9goAZTtYD6gFfu/2/9MDe6mTGcw= github.com/cs3org/go-cs3apis v0.0.0-20211104090126-8e972dca8304/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY= +github.com/cs3org/reva v1.16.1-0.20211203225713-939768a1af06 h1:NP+Zvli7+9USaDss/+Ywk4KJ0H7n82UHZiU4V+x25I0= +github.com/cs3org/reva v1.16.1-0.20211203225713-939768a1af06/go.mod h1:3n/zVKsKTCL10Mwn2Nhtvn50gP5mA+933lQ2IYNAJso= github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8 h1:Z9lwXumT5ACSmJ7WGnFl+OMLLjpz5uR2fyz7dC255FI= github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8/go.mod h1:4abs/jPXcmJzYoYGF91JF9Uq9s/KL5n1jvFDix8KcqY= github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= diff --git a/ocis-pkg/indexer/index/cs3/autoincrement.go b/ocis-pkg/indexer/index/cs3/autoincrement.go index cd88396ea2..81d3d8ab0d 100644 --- a/ocis-pkg/indexer/index/cs3/autoincrement.go +++ b/ocis-pkg/indexer/index/cs3/autoincrement.go @@ -271,8 +271,7 @@ func (idx *Autoincrement) resolveSymlink(name string) (string, error) { b, err := idx.metadataStorage.downloadHelper(ctx, name) if err != nil { - switch err.(type) { - case notFoundErr: + if IsNotFoundErr(err) { return "", os.ErrNotExist } return "", err diff --git a/ocis-pkg/indexer/index/cs3/non_unique.go b/ocis-pkg/indexer/index/cs3/non_unique.go index 9492886819..e1b49c5a95 100644 --- a/ocis-pkg/indexer/index/cs3/non_unique.go +++ b/ocis-pkg/indexer/index/cs3/non_unique.go @@ -327,8 +327,7 @@ func (idx *NonUnique) resolveSymlink(name string) (string, error) { b, err := idx.metadataStorage.downloadHelper(ctx, name) if err != nil { - switch err.(type) { - case notFoundErr: + if IsNotFoundErr(err) { return "", os.ErrNotExist } return "", err diff --git a/ocis-pkg/indexer/index/cs3/unique.go b/ocis-pkg/indexer/index/cs3/unique.go index cd9b56615d..aca3ff4582 100644 --- a/ocis-pkg/indexer/index/cs3/unique.go +++ b/ocis-pkg/indexer/index/cs3/unique.go @@ -280,8 +280,7 @@ func (idx *Unique) resolveSymlink(name string) (string, error) { b, err := idx.metadataStorage.downloadHelper(ctx, name) if err != nil { - switch err.(type) { - case notFoundErr: + if IsNotFoundErr(err) { return "", os.ErrNotExist } return "", err From 122822a87e5978efaa4958a009d69b8ea6d4349f Mon Sep 17 00:00:00 2001 From: Willy Kloucek Date: Mon, 6 Dec 2021 10:44:02 +0100 Subject: [PATCH 07/14] refactor auth --- accounts/pkg/storage/cs3.go | 35 +++++++++------------ ocis-pkg/indexer/index/cs3/autoincrement.go | 6 +--- ocis-pkg/indexer/index/cs3/non_unique.go | 6 +--- ocis-pkg/indexer/index/cs3/unique.go | 6 +--- 4 files changed, 18 insertions(+), 35 deletions(-) diff --git a/accounts/pkg/storage/cs3.go b/accounts/pkg/storage/cs3.go index d4d4321d08..824e2f7f03 100644 --- a/accounts/pkg/storage/cs3.go +++ b/accounts/pkg/storage/cs3.go @@ -62,12 +62,11 @@ func NewCS3Repo(cfg *config.Config) (Repo, error) { // WriteAccount writes an account via cs3 and modifies the provided account (e.g. with a generated id). func (r CS3Repo) WriteAccount(ctx context.Context, a *proto.Account) (err error) { - t, err := r.authenticate(ctx) + ctx, err = r.getAuthenticatedContext(ctx) if err != nil { return err } - ctx = metadata.AppendToOutgoingContext(ctx, revactx.TokenHeader, t) if err := r.makeRootDirIfNotExist(ctx, accountsFolder); err != nil { return err } @@ -84,23 +83,21 @@ func (r CS3Repo) WriteAccount(ctx context.Context, a *proto.Account) (err error) // LoadAccount loads an account via cs3 by id and writes it to the provided account func (r CS3Repo) LoadAccount(ctx context.Context, id string, a *proto.Account) (err error) { - t, err := r.authenticate(ctx) + ctx, err = r.getAuthenticatedContext(ctx) if err != nil { return err } - ctx = metadata.AppendToOutgoingContext(ctx, revactx.TokenHeader, t) return r.loadAccount(ctx, id, a) } // LoadAccounts loads all the accounts from the cs3 api func (r CS3Repo) LoadAccounts(ctx context.Context, a *[]*proto.Account) (err error) { - t, err := r.authenticate(ctx) + ctx, err = r.getAuthenticatedContext(ctx) if err != nil { return err } - ctx = metadata.AppendToOutgoingContext(ctx, revactx.TokenHeader, t) res, err := r.storageProvider.ListContainer(ctx, &provider.ListContainerRequest{ Ref: &provider.Reference{ Path: path.Join(storageMountPath, accountsFolder), @@ -136,13 +133,11 @@ func (r CS3Repo) loadAccount(ctx context.Context, id string, a *proto.Account) e // DeleteAccount deletes an account via cs3 by id func (r CS3Repo) DeleteAccount(ctx context.Context, id string) (err error) { - t, err := r.authenticate(ctx) + ctx, err = r.getAuthenticatedContext(ctx) if err != nil { return err } - ctx = metadata.AppendToOutgoingContext(ctx, revactx.TokenHeader, t) - resp, err := r.storageProvider.Delete(ctx, &provider.DeleteRequest{ Ref: &provider.Reference{ Path: path.Join(storageMountPath, accountsFolder, id), @@ -163,12 +158,11 @@ func (r CS3Repo) DeleteAccount(ctx context.Context, id string) (err error) { // WriteGroup writes a group via cs3 and modifies the provided group (e.g. with a generated id). func (r CS3Repo) WriteGroup(ctx context.Context, g *proto.Group) (err error) { - t, err := r.authenticate(ctx) + ctx, err = r.getAuthenticatedContext(ctx) if err != nil { return err } - ctx = metadata.AppendToOutgoingContext(ctx, revactx.TokenHeader, t) if err := r.makeRootDirIfNotExist(ctx, groupsFolder); err != nil { return err } @@ -184,23 +178,21 @@ func (r CS3Repo) WriteGroup(ctx context.Context, g *proto.Group) (err error) { // LoadGroup loads a group via cs3 by id and writes it to the provided group func (r CS3Repo) LoadGroup(ctx context.Context, id string, g *proto.Group) (err error) { - t, err := r.authenticate(ctx) + ctx, err = r.getAuthenticatedContext(ctx) if err != nil { return err } - ctx = metadata.AppendToOutgoingContext(ctx, revactx.TokenHeader, t) return r.loadGroup(ctx, id, g) } // LoadGroups loads all the groups from the cs3 api func (r CS3Repo) LoadGroups(ctx context.Context, g *[]*proto.Group) (err error) { - t, err := r.authenticate(ctx) + ctx, err = r.getAuthenticatedContext(ctx) if err != nil { return err } - ctx = metadata.AppendToOutgoingContext(ctx, revactx.TokenHeader, t) res, err := r.storageProvider.ListContainer(ctx, &provider.ListContainerRequest{ Ref: &provider.Reference{ Path: path.Join(storageMountPath, groupsFolder), @@ -236,13 +228,11 @@ func (r CS3Repo) loadGroup(ctx context.Context, id string, g *proto.Group) error // DeleteGroup deletes a group via cs3 by id func (r CS3Repo) DeleteGroup(ctx context.Context, id string) (err error) { - t, err := r.authenticate(ctx) + ctx, err = r.getAuthenticatedContext(ctx) if err != nil { return err } - ctx = metadata.AppendToOutgoingContext(ctx, revactx.TokenHeader, t) - resp, err := r.storageProvider.Delete(ctx, &provider.DeleteRequest{ Ref: &provider.Reference{ Path: path.Join(storageMountPath, groupsFolder, id), @@ -261,8 +251,13 @@ func (r CS3Repo) DeleteGroup(ctx context.Context, id string) (err error) { return err } -func (r CS3Repo) authenticate(ctx context.Context) (token string, err error) { - return AuthenticateCS3(ctx, r.cfg.ServiceUser, r.tm) +func (r CS3Repo) getAuthenticatedContext(ctx context.Context) (context.Context, error) { + t, err := AuthenticateCS3(ctx, r.cfg.ServiceUser, r.tm) + if err != nil { + return nil, err + } + ctx = metadata.AppendToOutgoingContext(ctx, revactx.TokenHeader, t) + return ctx, nil } // AuthenticateCS3 mints an auth token for communicating with cs3 storage based on a service user from config diff --git a/ocis-pkg/indexer/index/cs3/autoincrement.go b/ocis-pkg/indexer/index/cs3/autoincrement.go index 81d3d8ab0d..b84d5b06a1 100644 --- a/ocis-pkg/indexer/index/cs3/autoincrement.go +++ b/ocis-pkg/indexer/index/cs3/autoincrement.go @@ -289,10 +289,6 @@ func (idx *Autoincrement) makeDirIfNotExists(folder string) error { return storage.MakeDirIfNotExist(ctx, idx.metadataStorage.storageProvider, folder) } -func (idx *Autoincrement) authenticate(ctx context.Context) (token string, err error) { - return storage.AuthenticateCS3(ctx, idx.cs3conf.ServiceUser, idx.metadataStorage.tokenManager) -} - func (idx *Autoincrement) next() (int, error) { ctx, err := idx.getAuthenticatedContext(context.Background()) if err != nil { @@ -333,7 +329,7 @@ func (idx *Autoincrement) next() (int, error) { } func (idx *Autoincrement) getAuthenticatedContext(ctx context.Context) (context.Context, error) { - t, err := idx.authenticate(ctx) + t, err := storage.AuthenticateCS3(ctx, idx.cs3conf.ServiceUser, idx.metadataStorage.tokenManager) if err != nil { return nil, err } diff --git a/ocis-pkg/indexer/index/cs3/non_unique.go b/ocis-pkg/indexer/index/cs3/non_unique.go index e1b49c5a95..d41109903d 100644 --- a/ocis-pkg/indexer/index/cs3/non_unique.go +++ b/ocis-pkg/indexer/index/cs3/non_unique.go @@ -337,7 +337,7 @@ func (idx *NonUnique) resolveSymlink(name string) (string, error) { } func (idx *NonUnique) getAuthenticatedContext(ctx context.Context) (context.Context, error) { - t, err := idx.authenticate(ctx) + t, err := storage.AuthenticateCS3(ctx, idx.cs3conf.ServiceUser, idx.metadataStorage.tokenManager) if err != nil { return nil, err } @@ -354,7 +354,3 @@ func (idx *NonUnique) Delete() error { return deleteIndexRoot(ctx, idx.metadataStorage.storageProvider, idx.indexRootDir) } - -func (idx *NonUnique) authenticate(ctx context.Context) (token string, err error) { - return storage.AuthenticateCS3(ctx, idx.cs3conf.ServiceUser, idx.metadataStorage.tokenManager) -} diff --git a/ocis-pkg/indexer/index/cs3/unique.go b/ocis-pkg/indexer/index/cs3/unique.go index aca3ff4582..9ab0a91477 100644 --- a/ocis-pkg/indexer/index/cs3/unique.go +++ b/ocis-pkg/indexer/index/cs3/unique.go @@ -297,12 +297,8 @@ func (idx *Unique) makeDirIfNotExists(folder string) error { return storage.MakeDirIfNotExist(ctx, idx.metadataStorage.storageProvider, folder) } -func (idx *Unique) authenticate(ctx context.Context) (token string, err error) { - return storage.AuthenticateCS3(ctx, idx.cs3conf.ServiceUser, idx.metadataStorage.tokenManager) -} - func (idx *Unique) getAuthenticatedContext(ctx context.Context) (context.Context, error) { - t, err := idx.authenticate(ctx) + t, err := storage.AuthenticateCS3(ctx, idx.cs3conf.ServiceUser, idx.metadataStorage.tokenManager) if err != nil { return nil, err } From 81408a3568243d2f19a5883160856c157b82c4ba Mon Sep 17 00:00:00 2001 From: Willy Kloucek Date: Mon, 6 Dec 2021 12:00:03 +0100 Subject: [PATCH 08/14] move up and download to a separate package --- accounts/pkg/storage/cs3.go | 131 +++--------------- ocis-pkg/indexer/index/cs3/autoincrement.go | 38 ++--- ocis-pkg/indexer/index/cs3/non_unique.go | 42 +++--- ocis-pkg/indexer/index/cs3/unique.go | 33 +++-- ocis-pkg/metadata_storage/errors.go | 17 +++ .../metadata_storage.go} | 41 +++--- 6 files changed, 120 insertions(+), 182 deletions(-) create mode 100644 ocis-pkg/metadata_storage/errors.go rename ocis-pkg/{indexer/index/cs3/storage.go => metadata_storage/metadata_storage.go} (81%) diff --git a/accounts/pkg/storage/cs3.go b/accounts/pkg/storage/cs3.go index 824e2f7f03..f628f538f4 100644 --- a/accounts/pkg/storage/cs3.go +++ b/accounts/pkg/storage/cs3.go @@ -1,12 +1,8 @@ package storage import ( - "bytes" "context" "encoding/json" - "errors" - "io/ioutil" - "net/http" "path" "path/filepath" @@ -22,6 +18,7 @@ import ( "github.com/owncloud/ocis/accounts/pkg/config" "github.com/owncloud/ocis/accounts/pkg/proto/v0" olog "github.com/owncloud/ocis/ocis-pkg/log" + metadatastorage "github.com/owncloud/ocis/ocis-pkg/metadata_storage" "google.golang.org/grpc/metadata" ) @@ -31,10 +28,10 @@ const ( // CS3Repo provides a cs3 implementation of the Repo interface type CS3Repo struct { - cfg *config.Config - tm token.Manager - storageProvider provider.ProviderAPIClient - dataGatewayClient *http.Client + cfg *config.Config + tm token.Manager + storageProvider provider.ProviderAPIClient + metadataStorage metadatastorage.MetadataStorage } // NewCS3Repo creates a new cs3 repo @@ -52,11 +49,16 @@ func NewCS3Repo(cfg *config.Config) (Repo, error) { return nil, err } + ms, err := metadatastorage.NewMetadataStorage(cfg.Repo.CS3.ProviderAddr) + if err != nil { + return nil, err + } + return CS3Repo{ - cfg: cfg, - tm: tokenManager, - storageProvider: client, - dataGatewayClient: http.DefaultClient, + cfg: cfg, + tm: tokenManager, + storageProvider: client, + metadataStorage: ms, }, nil } @@ -76,7 +78,7 @@ func (r CS3Repo) WriteAccount(ctx context.Context, a *proto.Account) (err error) return err } - err = r.uploadHelper(ctx, r.accountURL(a.Id), by) + err = r.metadataStorage.SimpleUpload(ctx, r.accountURL(a.Id), by) return err } @@ -121,9 +123,9 @@ func (r CS3Repo) LoadAccounts(ctx context.Context, a *[]*proto.Account) (err err } func (r CS3Repo) loadAccount(ctx context.Context, id string, a *proto.Account) error { - account, err := r.downloadHelper(ctx, r.accountURL(id)) + account, err := r.metadataStorage.SimpleDownload(ctx, r.accountURL(id)) if err != nil { - if IsNotFoundErr(err) { + if metadatastorage.IsNotFoundErr(err) { return ¬FoundErr{"account", id} } return err @@ -172,7 +174,7 @@ func (r CS3Repo) WriteGroup(ctx context.Context, g *proto.Group) (err error) { return err } - err = r.uploadHelper(ctx, r.groupURL(g.Id), by) + err = r.metadataStorage.SimpleUpload(ctx, r.groupURL(g.Id), by) return err } @@ -216,9 +218,9 @@ func (r CS3Repo) LoadGroups(ctx context.Context, g *[]*proto.Group) (err error) } func (r CS3Repo) loadGroup(ctx context.Context, id string, g *proto.Group) error { - group, err := r.downloadHelper(ctx, r.groupURL(id)) + group, err := r.metadataStorage.SimpleDownload(ctx, r.groupURL(id)) if err != nil { - if IsNotFoundErr(err) { + if metadatastorage.IsNotFoundErr(err) { return ¬FoundErr{"group", id} } return err @@ -316,96 +318,3 @@ func MakeDirIfNotExist(ctx context.Context, sp provider.ProviderAPIClient, folde return nil } - -func (r CS3Repo) uploadHelper(ctx context.Context, uploadpath string, content []byte) error { - - ref := provider.InitiateFileUploadRequest{ - Ref: &provider.Reference{ - Path: path.Join(storageMountPath, uploadpath), - }, - } - - res, err := r.storageProvider.InitiateFileUpload(ctx, &ref) - if err != nil { - return err - } - - var endpoint string - - for _, proto := range res.GetProtocols() { - if proto.Protocol == "simple" { - endpoint = proto.GetUploadEndpoint() - break - } - } - if endpoint == "" { - return errors.New("metadata storage doesn't support the simple upload protocol") - } - - req, err := http.NewRequest(http.MethodPut, endpoint, bytes.NewReader(content)) - if err != nil { - return err - } - md, _ := metadata.FromOutgoingContext(ctx) - req.Header.Add(revactx.TokenHeader, md.Get(revactx.TokenHeader)[0]) - resp, err := r.dataGatewayClient.Do(req) - if err != nil { - return err - } - if err = resp.Body.Close(); err != nil { - return err - } - return nil -} - -func (r CS3Repo) downloadHelper(ctx context.Context, downloadpath string) (content []byte, err error) { - - ref := provider.InitiateFileDownloadRequest{ - Ref: &provider.Reference{ - Path: path.Join(storageMountPath, downloadpath), - }, - } - - res, err := r.storageProvider.InitiateFileDownload(ctx, &ref) - if err != nil { - return []byte{}, err - } - - var endpoint string - - for _, proto := range res.GetProtocols() { - if proto.Protocol == "simple" { - endpoint = proto.GetDownloadEndpoint() - break - } - } - if endpoint == "" { - return []byte{}, errors.New("metadata storage doesn't support the simple download protocol") - } - - req, err := http.NewRequest(http.MethodGet, endpoint, nil) - if err != nil { - return []byte{}, err - } - md, _ := metadata.FromOutgoingContext(ctx) - req.Header.Add(revactx.TokenHeader, md.Get(revactx.TokenHeader)[0]) - resp, err := r.dataGatewayClient.Do(req) - if err != nil { - return []byte{}, err - } - - if resp.StatusCode != http.StatusOK { - return []byte{}, ¬FoundErr{} - } - - b, err := ioutil.ReadAll(resp.Body) - if err != nil { - return []byte{}, err - } - - if err = resp.Body.Close(); err != nil { - return []byte{}, err - } - - return b, nil -} diff --git a/ocis-pkg/indexer/index/cs3/autoincrement.go b/ocis-pkg/indexer/index/cs3/autoincrement.go index b84d5b06a1..1b223a7ed0 100644 --- a/ocis-pkg/indexer/index/cs3/autoincrement.go +++ b/ocis-pkg/indexer/index/cs3/autoincrement.go @@ -2,7 +2,6 @@ package cs3 import ( "context" - "net/http" "os" "path" "path/filepath" @@ -18,12 +17,14 @@ import ( provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" revactx "github.com/cs3org/reva/pkg/ctx" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" + "github.com/cs3org/reva/pkg/token" "github.com/cs3org/reva/pkg/token/manager/jwt" "google.golang.org/grpc/metadata" "github.com/owncloud/ocis/ocis-pkg/indexer/index" "github.com/owncloud/ocis/ocis-pkg/indexer/option" "github.com/owncloud/ocis/ocis-pkg/indexer/registry" + metadatastorage "github.com/owncloud/ocis/ocis-pkg/metadata_storage" ) // Autoincrement are fields for an index of type autoincrement. @@ -34,7 +35,9 @@ type Autoincrement struct { indexBaseDir string indexRootDir string - metadataStorage *metadataStorage + tokenManager token.Manager + storageProvider provider.ProviderAPIClient + metadataStorage *metadatastorage.MetadataStorage cs3conf *Config bound *option.Bound @@ -73,21 +76,22 @@ func (idx *Autoincrement) Init() error { tokenManager, err := jwt.New(map[string]interface{}{ "secret": idx.cs3conf.JWTSecret, }) - if err != nil { return err } + idx.tokenManager = tokenManager client, err := pool.GetStorageProviderServiceClient(idx.cs3conf.ProviderAddr) if err != nil { return err } + idx.storageProvider = client - idx.metadataStorage = &metadataStorage{ - tokenManager: tokenManager, - storageProvider: client, - dataGatewayClient: http.DefaultClient, + m, err := metadatastorage.NewMetadataStorage(idx.cs3conf.ProviderAddr) + if err != nil { + return err } + idx.metadataStorage = &m if err := idx.makeDirIfNotExists(idx.indexBaseDir); err != nil { return err @@ -159,7 +163,7 @@ func (idx *Autoincrement) Remove(id string, v string) error { } deletePath := path.Join("/meta", idx.indexRootDir, v) - resp, err := idx.metadataStorage.storageProvider.Delete(ctx, &provider.DeleteRequest{ + resp, err := idx.storageProvider.Delete(ctx, &provider.DeleteRequest{ Ref: &provider.Reference{ Path: deletePath, }, @@ -197,7 +201,7 @@ func (idx *Autoincrement) Search(pattern string) ([]string, error) { return nil, err } - res, err := idx.metadataStorage.storageProvider.ListContainer(ctx, &provider.ListContainerRequest{ + res, err := idx.storageProvider.ListContainer(ctx, &provider.ListContainerRequest{ Ref: &provider.Reference{ Path: path.Join("/meta", idx.indexRootDir), }, @@ -256,7 +260,7 @@ func (idx *Autoincrement) createSymlink(oldname, newname string) error { return os.ErrExist } - err = idx.metadataStorage.uploadHelper(ctx, newname, []byte(oldname)) + err = idx.metadataStorage.SimpleUpload(ctx, newname, []byte(oldname)) if err != nil { return err } @@ -269,9 +273,9 @@ func (idx *Autoincrement) resolveSymlink(name string) (string, error) { return "", err } - b, err := idx.metadataStorage.downloadHelper(ctx, name) + b, err := idx.metadataStorage.SimpleDownload(ctx, name) if err != nil { - if IsNotFoundErr(err) { + if metadatastorage.IsNotFoundErr(err) { return "", os.ErrNotExist } return "", err @@ -286,7 +290,7 @@ func (idx *Autoincrement) makeDirIfNotExists(folder string) error { return err } - return storage.MakeDirIfNotExist(ctx, idx.metadataStorage.storageProvider, folder) + return storage.MakeDirIfNotExist(ctx, idx.storageProvider, folder) } func (idx *Autoincrement) next() (int, error) { @@ -295,9 +299,9 @@ func (idx *Autoincrement) next() (int, error) { return -1, err } - res, err := idx.metadataStorage.storageProvider.ListContainer(ctx, &provider.ListContainerRequest{ + res, err := idx.storageProvider.ListContainer(ctx, &provider.ListContainerRequest{ Ref: &provider.Reference{ - Path: path.Join("/meta", idx.indexRootDir), + Path: path.Join("/meta", idx.indexRootDir), //TODO: }, }) @@ -329,7 +333,7 @@ func (idx *Autoincrement) next() (int, error) { } func (idx *Autoincrement) getAuthenticatedContext(ctx context.Context) (context.Context, error) { - t, err := storage.AuthenticateCS3(ctx, idx.cs3conf.ServiceUser, idx.metadataStorage.tokenManager) + t, err := storage.AuthenticateCS3(ctx, idx.cs3conf.ServiceUser, idx.tokenManager) if err != nil { return nil, err } @@ -344,5 +348,5 @@ func (idx *Autoincrement) Delete() error { return err } - return deleteIndexRoot(ctx, idx.metadataStorage.storageProvider, idx.indexRootDir) + return deleteIndexRoot(ctx, idx.storageProvider, idx.indexRootDir) } diff --git a/ocis-pkg/indexer/index/cs3/non_unique.go b/ocis-pkg/indexer/index/cs3/non_unique.go index d41109903d..8976199fc5 100644 --- a/ocis-pkg/indexer/index/cs3/non_unique.go +++ b/ocis-pkg/indexer/index/cs3/non_unique.go @@ -2,7 +2,6 @@ package cs3 import ( "context" - "net/http" "os" "path" "path/filepath" @@ -14,11 +13,13 @@ import ( provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" revactx "github.com/cs3org/reva/pkg/ctx" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" + "github.com/cs3org/reva/pkg/token" "github.com/cs3org/reva/pkg/token/manager/jwt" idxerrs "github.com/owncloud/ocis/ocis-pkg/indexer/errors" "github.com/owncloud/ocis/ocis-pkg/indexer/index" "github.com/owncloud/ocis/ocis-pkg/indexer/option" "github.com/owncloud/ocis/ocis-pkg/indexer/registry" + metadatastorage "github.com/owncloud/ocis/ocis-pkg/metadata_storage" "google.golang.org/grpc/metadata" ) @@ -35,7 +36,9 @@ type NonUnique struct { indexBaseDir string indexRootDir string - metadataStorage *metadataStorage + tokenManager token.Manager + storageProvider provider.ProviderAPIClient + metadataStorage *metadatastorage.MetadataStorage cs3conf *Config } @@ -75,21 +78,22 @@ func (idx *NonUnique) Init() error { tokenManager, err := jwt.New(map[string]interface{}{ "secret": idx.cs3conf.JWTSecret, }) - if err != nil { return err } + idx.tokenManager = tokenManager client, err := pool.GetStorageProviderServiceClient(idx.cs3conf.ProviderAddr) if err != nil { return err } + idx.storageProvider = client - idx.metadataStorage = &metadataStorage{ - tokenManager: tokenManager, - storageProvider: client, - dataGatewayClient: http.DefaultClient, + m, err := metadatastorage.NewMetadataStorage(idx.cs3conf.ProviderAddr) + if err != nil { + return err } + idx.metadataStorage = &m if err := idx.makeDirIfNotExists(idx.indexBaseDir); err != nil { return err @@ -113,7 +117,7 @@ func (idx *NonUnique) Lookup(v string) ([]string, error) { return nil, err } - res, err := idx.metadataStorage.storageProvider.ListContainer(ctx, &provider.ListContainerRequest{ + res, err := idx.storageProvider.ListContainer(ctx, &provider.ListContainerRequest{ Ref: &provider.Reference{ Path: path.Join("/meta", idx.indexRootDir, v), }, @@ -169,7 +173,7 @@ func (idx *NonUnique) Remove(id string, v string) error { } deletePath := path.Join("/meta", idx.indexRootDir, v, id) - resp, err := idx.metadataStorage.storageProvider.Delete(ctx, &provider.DeleteRequest{ + resp, err := idx.storageProvider.Delete(ctx, &provider.DeleteRequest{ Ref: &provider.Reference{ Path: deletePath, }, @@ -184,7 +188,7 @@ func (idx *NonUnique) Remove(id string, v string) error { } toStat := path.Join("/meta", idx.indexRootDir, v) - lcResp, err := idx.metadataStorage.storageProvider.ListContainer(ctx, &provider.ListContainerRequest{ + lcResp, err := idx.storageProvider.ListContainer(ctx, &provider.ListContainerRequest{ Ref: &provider.Reference{ Path: toStat, }, @@ -195,7 +199,7 @@ func (idx *NonUnique) Remove(id string, v string) error { if len(lcResp.Infos) == 0 { deletePath = path.Join("/meta", idx.indexRootDir, v) - _, err := idx.metadataStorage.storageProvider.Delete(ctx, &provider.DeleteRequest{ + _, err := idx.storageProvider.Delete(ctx, &provider.DeleteRequest{ Ref: &provider.Reference{ Path: deletePath, }, @@ -239,7 +243,7 @@ func (idx *NonUnique) Search(pattern string) ([]string, error) { foldersMatched := make([]string, 0) matches := make([]string, 0) - res, err := idx.metadataStorage.storageProvider.ListContainer(ctx, &provider.ListContainerRequest{ + res, err := idx.storageProvider.ListContainer(ctx, &provider.ListContainerRequest{ Ref: &provider.Reference{ Path: path.Join("/meta", idx.indexRootDir), }, @@ -260,7 +264,7 @@ func (idx *NonUnique) Search(pattern string) ([]string, error) { } for i := range foldersMatched { - res, _ := idx.metadataStorage.storageProvider.ListContainer(ctx, &provider.ListContainerRequest{ + res, _ := idx.storageProvider.ListContainer(ctx, &provider.ListContainerRequest{ Ref: &provider.Reference{ Path: foldersMatched[i], }, @@ -299,7 +303,7 @@ func (idx *NonUnique) makeDirIfNotExists(folder string) error { if err != nil { return err } - return storage.MakeDirIfNotExist(ctx, idx.metadataStorage.storageProvider, folder) + return storage.MakeDirIfNotExist(ctx, idx.storageProvider, folder) } func (idx *NonUnique) createSymlink(oldname, newname string) error { @@ -312,7 +316,7 @@ func (idx *NonUnique) createSymlink(oldname, newname string) error { return os.ErrExist } - err = idx.metadataStorage.uploadHelper(ctx, newname, []byte(oldname)) + err = idx.metadataStorage.SimpleUpload(ctx, newname, []byte(oldname)) if err != nil { return err } @@ -325,9 +329,9 @@ func (idx *NonUnique) resolveSymlink(name string) (string, error) { return "", err } - b, err := idx.metadataStorage.downloadHelper(ctx, name) + b, err := idx.metadataStorage.SimpleDownload(ctx, name) if err != nil { - if IsNotFoundErr(err) { + if metadatastorage.IsNotFoundErr(err) { return "", os.ErrNotExist } return "", err @@ -337,7 +341,7 @@ func (idx *NonUnique) resolveSymlink(name string) (string, error) { } func (idx *NonUnique) getAuthenticatedContext(ctx context.Context) (context.Context, error) { - t, err := storage.AuthenticateCS3(ctx, idx.cs3conf.ServiceUser, idx.metadataStorage.tokenManager) + t, err := storage.AuthenticateCS3(ctx, idx.cs3conf.ServiceUser, idx.tokenManager) if err != nil { return nil, err } @@ -352,5 +356,5 @@ func (idx *NonUnique) Delete() error { return err } - return deleteIndexRoot(ctx, idx.metadataStorage.storageProvider, idx.indexRootDir) + return deleteIndexRoot(ctx, idx.storageProvider, idx.indexRootDir) } diff --git a/ocis-pkg/indexer/index/cs3/unique.go b/ocis-pkg/indexer/index/cs3/unique.go index 9ab0a91477..7114d4fcf5 100644 --- a/ocis-pkg/indexer/index/cs3/unique.go +++ b/ocis-pkg/indexer/index/cs3/unique.go @@ -2,7 +2,6 @@ package cs3 import ( "context" - "net/http" "os" "path" "path/filepath" @@ -14,11 +13,13 @@ import ( provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" revactx "github.com/cs3org/reva/pkg/ctx" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" + "github.com/cs3org/reva/pkg/token" "github.com/cs3org/reva/pkg/token/manager/jwt" idxerrs "github.com/owncloud/ocis/ocis-pkg/indexer/errors" "github.com/owncloud/ocis/ocis-pkg/indexer/index" "github.com/owncloud/ocis/ocis-pkg/indexer/option" "github.com/owncloud/ocis/ocis-pkg/indexer/registry" + metadatastorage "github.com/owncloud/ocis/ocis-pkg/metadata_storage" "google.golang.org/grpc/metadata" ) @@ -31,7 +32,9 @@ type Unique struct { indexBaseDir string indexRootDir string - metadataStorage *metadataStorage + tokenManager token.Manager + storageProvider provider.ProviderAPIClient + metadataStorage *metadatastorage.MetadataStorage cs3conf *Config } @@ -73,17 +76,19 @@ func (idx *Unique) Init() error { if err != nil { return err } + idx.tokenManager = tokenManager client, err := pool.GetStorageProviderServiceClient(idx.cs3conf.ProviderAddr) if err != nil { return err } + idx.storageProvider = client - idx.metadataStorage = &metadataStorage{ - tokenManager: tokenManager, - storageProvider: client, - dataGatewayClient: http.DefaultClient, + m, err := metadatastorage.NewMetadataStorage(idx.cs3conf.ProviderAddr) + if err != nil { + return err } + idx.metadataStorage = &m if err := idx.makeDirIfNotExists(idx.indexBaseDir); err != nil { return err @@ -158,7 +163,7 @@ func (idx *Unique) Remove(id string, v string) error { } deletePath := path.Join("/meta", idx.indexRootDir, v) - resp, err := idx.metadataStorage.storageProvider.Delete(ctx, &provider.DeleteRequest{ + resp, err := idx.storageProvider.Delete(ctx, &provider.DeleteRequest{ Ref: &provider.Reference{ Path: deletePath, }, @@ -205,7 +210,7 @@ func (idx *Unique) Search(pattern string) ([]string, error) { return nil, err } - res, err := idx.metadataStorage.storageProvider.ListContainer(ctx, &provider.ListContainerRequest{ + res, err := idx.storageProvider.ListContainer(ctx, &provider.ListContainerRequest{ Ref: &provider.Reference{ Path: path.Join("/meta", idx.indexRootDir), }, @@ -264,7 +269,7 @@ func (idx *Unique) createSymlink(oldname, newname string) error { return os.ErrExist } - err = idx.metadataStorage.uploadHelper(ctx, newname, []byte(oldname)) + err = idx.metadataStorage.SimpleUpload(ctx, newname, []byte(oldname)) if err != nil { return err } @@ -278,9 +283,9 @@ func (idx *Unique) resolveSymlink(name string) (string, error) { return "", err } - b, err := idx.metadataStorage.downloadHelper(ctx, name) + b, err := idx.metadataStorage.SimpleDownload(ctx, name) if err != nil { - if IsNotFoundErr(err) { + if metadatastorage.IsNotFoundErr(err) { return "", os.ErrNotExist } return "", err @@ -294,11 +299,11 @@ func (idx *Unique) makeDirIfNotExists(folder string) error { if err != nil { return err } - return storage.MakeDirIfNotExist(ctx, idx.metadataStorage.storageProvider, folder) + return storage.MakeDirIfNotExist(ctx, idx.storageProvider, folder) } func (idx *Unique) getAuthenticatedContext(ctx context.Context) (context.Context, error) { - t, err := storage.AuthenticateCS3(ctx, idx.cs3conf.ServiceUser, idx.metadataStorage.tokenManager) + t, err := storage.AuthenticateCS3(ctx, idx.cs3conf.ServiceUser, idx.tokenManager) if err != nil { return nil, err } @@ -313,5 +318,5 @@ func (idx *Unique) Delete() error { return err } - return deleteIndexRoot(ctx, idx.metadataStorage.storageProvider, idx.indexRootDir) + return deleteIndexRoot(ctx, idx.storageProvider, idx.indexRootDir) } diff --git a/ocis-pkg/metadata_storage/errors.go b/ocis-pkg/metadata_storage/errors.go new file mode 100644 index 0000000000..276207e505 --- /dev/null +++ b/ocis-pkg/metadata_storage/errors.go @@ -0,0 +1,17 @@ +package metadataStorage + +import "fmt" + +type notFoundErr struct { + typ, id string +} + +func (e notFoundErr) Error() string { + return fmt.Sprintf("%s with id %s not found", e.typ, e.id) +} + +// IsNotFoundErr can be returned by repo Load and Delete operations +func IsNotFoundErr(e error) bool { + _, ok := e.(*notFoundErr) + return ok +} diff --git a/ocis-pkg/indexer/index/cs3/storage.go b/ocis-pkg/metadata_storage/metadata_storage.go similarity index 81% rename from ocis-pkg/indexer/index/cs3/storage.go rename to ocis-pkg/metadata_storage/metadata_storage.go index 012e2c023c..bb7527426a 100644 --- a/ocis-pkg/indexer/index/cs3/storage.go +++ b/ocis-pkg/metadata_storage/metadata_storage.go @@ -1,17 +1,16 @@ -package cs3 +package metadataStorage import ( "bytes" "context" "errors" - "fmt" "io/ioutil" "net/http" "path" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" revactx "github.com/cs3org/reva/pkg/ctx" - "github.com/cs3org/reva/pkg/token" + "github.com/cs3org/reva/pkg/rgrpc/todo/pool" "google.golang.org/grpc/metadata" ) @@ -19,13 +18,27 @@ const ( storageMountPath = "/meta" ) -type metadataStorage struct { - tokenManager token.Manager +func NewMetadataStorage(providerAddr string) (s MetadataStorage, err error) { + + p, err := pool.GetStorageProviderServiceClient(providerAddr) + if err != nil { + return MetadataStorage{}, err + } + + c := http.DefaultClient + + return MetadataStorage{ + storageProvider: p, + dataGatewayClient: c, + }, nil +} + +type MetadataStorage struct { storageProvider provider.ProviderAPIClient dataGatewayClient *http.Client } -func (r metadataStorage) uploadHelper(ctx context.Context, uploadpath string, content []byte) error { +func (r MetadataStorage) SimpleUpload(ctx context.Context, uploadpath string, content []byte) error { ref := provider.InitiateFileUploadRequest{ Ref: &provider.Reference{ @@ -67,7 +80,7 @@ func (r metadataStorage) uploadHelper(ctx context.Context, uploadpath string, co return nil } -func (r metadataStorage) downloadHelper(ctx context.Context, downloadpath string) (content []byte, err error) { +func (r MetadataStorage) SimpleDownload(ctx context.Context, downloadpath string) (content []byte, err error) { ref := provider.InitiateFileDownloadRequest{ Ref: &provider.Reference{ Path: path.Join(storageMountPath, downloadpath), @@ -118,17 +131,3 @@ func (r metadataStorage) downloadHelper(ctx context.Context, downloadpath string return b, nil } - -type notFoundErr struct { - typ, id string -} - -func (e notFoundErr) Error() string { - return fmt.Sprintf("%s with id %s not found", e.typ, e.id) -} - -// IsNotFoundErr can be returned by repo Load and Delete operations -func IsNotFoundErr(e error) bool { - _, ok := e.(*notFoundErr) - return ok -} From ad62e230a61247c438183210a7dd9d7d9ab290e8 Mon Sep 17 00:00:00 2001 From: Willy Kloucek Date: Mon, 6 Dec 2021 13:26:28 +0100 Subject: [PATCH 09/14] fix add to groups endpoint --- ocs/pkg/service/v0/groups.go | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/ocs/pkg/service/v0/groups.go b/ocs/pkg/service/v0/groups.go index 5d89cb9f80..c9eee973c5 100644 --- a/ocs/pkg/service/v0/groups.go +++ b/ocs/pkg/service/v0/groups.go @@ -102,14 +102,8 @@ func (o Ocs) ListUserGroups(w http.ResponseWriter, r *http.Request) { // AddToGroup adds a user to a group func (o Ocs) AddToGroup(w http.ResponseWriter, r *http.Request) { - err := r.ParseForm() - if err != nil { - o.mustRender(w, r, response.ErrRender(data.MetaBadRequest.StatusCode, "Could not parse form from request")) - return - } - + groupid := r.PostFormValue("groupid") userid := chi.URLParam(r, "userid") - groupid := r.PostForm.Get("groupid") if groupid == "" { o.mustRender(w, r, response.ErrRender(data.MetaBadRequest.StatusCode, "empty group assignment: unspecified group")) From 60d3962ee4566d01e1eff1b4f1f2fb69c129ea3a Mon Sep 17 00:00:00 2001 From: Willy Kloucek Date: Mon, 6 Dec 2021 14:54:23 +0100 Subject: [PATCH 10/14] try without changed paths --- .drone.star | 10 +++++----- proxy/pkg/config/config.go | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.drone.star b/.drone.star index b912505fcc..44f0363ef7 100644 --- a/.drone.star +++ b/.drone.star @@ -1449,11 +1449,11 @@ def ocisServer(storage, accounts_hash_difficulty = 4, volumes = []): "OCIS_URL": "https://ocis-server:9200", "STORAGE_HOME_DRIVER": "%s" % (storage), "STORAGE_USERS_DRIVER": "%s" % (storage), - "STORAGE_USERS_DRIVER_LOCAL_ROOT": "/srv/app/tmp/ocis/local/root", - "STORAGE_USERS_DRIVER_OWNCLOUD_DATADIR": "/srv/app/tmp/ocis/owncloud/data", - "STORAGE_USERS_DRIVER_OCIS_ROOT": "/srv/app/tmp/ocis/storage/users", - "STORAGE_METADATA_DRIVER_OCIS_ROOT": "/srv/app/tmp/ocis/storage/metadata", - "STORAGE_SHARING_USER_JSON_FILE": "/srv/app/tmp/ocis/shares.json", + #"STORAGE_USERS_DRIVER_LOCAL_ROOT": "/srv/app/tmp/ocis/local/root", + #"STORAGE_USERS_DRIVER_OWNCLOUD_DATADIR": "/srv/app/tmp/ocis/owncloud/data", + #"STORAGE_USERS_DRIVER_OCIS_ROOT": "/srv/app/tmp/ocis/storage/users", + #"STORAGE_METADATA_DRIVER_OCIS_ROOT": "/srv/app/tmp/ocis/storage/metadata", + #"STORAGE_SHARING_USER_JSON_FILE": "/srv/app/tmp/ocis/shares.json", "PROXY_ENABLE_BASIC_AUTH": True, "WEB_UI_CONFIG": "/drone/src/tests/config/drone/ocis-config.json", "IDP_IDENTIFIER_REGISTRATION_CONF": "/drone/src/tests/config/drone/identifier-registration.yml", diff --git a/proxy/pkg/config/config.go b/proxy/pkg/config/config.go index bd21ee1e32..7a8b361487 100644 --- a/proxy/pkg/config/config.go +++ b/proxy/pkg/config/config.go @@ -203,7 +203,7 @@ func New() *Config { func DefaultConfig() *Config { return &Config{ Debug: Debug{ - Addr: "0.0.0.0:9205", + Addr: "127.0.0.1:9205", Token: "", }, HTTP: HTTP{ From 9109f8843ae87112f02531e86e410b8249cdd3ef Mon Sep 17 00:00:00 2001 From: Willy Kloucek Date: Mon, 6 Dec 2021 17:04:12 +0100 Subject: [PATCH 11/14] always unescape url params --- graph/pkg/service/v0/groups.go | 6 ++++++ graph/pkg/service/v0/users.go | 6 ++++++ ocs/pkg/service/v0/groups.go | 23 ++++++++++++++++++++--- ocs/pkg/service/v0/users.go | 26 +++++++++++++++++++++----- 4 files changed, 53 insertions(+), 8 deletions(-) diff --git a/graph/pkg/service/v0/groups.go b/graph/pkg/service/v0/groups.go index 01af9178cc..0598126368 100644 --- a/graph/pkg/service/v0/groups.go +++ b/graph/pkg/service/v0/groups.go @@ -3,6 +3,7 @@ package svc import ( "errors" "net/http" + "net/url" "github.com/owncloud/ocis/graph/pkg/service/v0/errorcode" @@ -30,6 +31,11 @@ func (g Graph) GetGroups(w http.ResponseWriter, r *http.Request) { // GetGroup implements the Service interface. func (g Graph) GetGroup(w http.ResponseWriter, r *http.Request) { groupID := chi.URLParam(r, "groupID") + groupID, err := url.PathUnescape(groupID) + if err != nil { + errorcode.InvalidRequest.Render(w, r, http.StatusBadRequest, "unescaping group id failed") + } + if groupID == "" { errorcode.InvalidRequest.Render(w, r, http.StatusBadRequest, "missing group id") return diff --git a/graph/pkg/service/v0/users.go b/graph/pkg/service/v0/users.go index 8c73736dc3..c62337cbc1 100644 --- a/graph/pkg/service/v0/users.go +++ b/graph/pkg/service/v0/users.go @@ -3,6 +3,7 @@ package svc import ( "errors" "net/http" + "net/url" revactx "github.com/cs3org/reva/pkg/ctx" "github.com/go-chi/chi/v5" @@ -49,6 +50,11 @@ func (g Graph) GetUsers(w http.ResponseWriter, r *http.Request) { // GetUser implements the Service interface. func (g Graph) GetUser(w http.ResponseWriter, r *http.Request) { userID := chi.URLParam(r, "userID") + userID, err := url.PathUnescape(userID) + if err != nil { + errorcode.InvalidRequest.Render(w, r, http.StatusBadRequest, "unescaping user id failed") + } + if userID == "" { errorcode.InvalidRequest.Render(w, r, http.StatusBadRequest, "missing user id") return diff --git a/ocs/pkg/service/v0/groups.go b/ocs/pkg/service/v0/groups.go index c9eee973c5..fd24e2b7bf 100644 --- a/ocs/pkg/service/v0/groups.go +++ b/ocs/pkg/service/v0/groups.go @@ -22,8 +22,11 @@ import ( // ListUserGroups lists a users groups func (o Ocs) ListUserGroups(w http.ResponseWriter, r *http.Request) { userid := chi.URLParam(r, "userid") + userid, err := url.PathUnescape(userid) + if err != nil { + o.mustRender(w, r, response.ErrRender(data.MetaServerError.StatusCode, err.Error())) + } var account *accounts.Account - var err error // short circuit if there is a user already in the context if u, ok := revactx.ContextGetUser(r.Context()); ok { @@ -104,6 +107,10 @@ func (o Ocs) ListUserGroups(w http.ResponseWriter, r *http.Request) { func (o Ocs) AddToGroup(w http.ResponseWriter, r *http.Request) { groupid := r.PostFormValue("groupid") userid := chi.URLParam(r, "userid") + userid, err := url.PathUnescape(userid) + if err != nil { + o.mustRender(w, r, response.ErrRender(data.MetaServerError.StatusCode, err.Error())) + } if groupid == "" { o.mustRender(w, r, response.ErrRender(data.MetaBadRequest.StatusCode, "empty group assignment: unspecified group")) @@ -155,8 +162,10 @@ func (o Ocs) AddToGroup(w http.ResponseWriter, r *http.Request) { // RemoveFromGroup removes a user from a group func (o Ocs) RemoveFromGroup(w http.ResponseWriter, r *http.Request) { userid := chi.URLParam(r, "userid") - - var err error + userid, err := url.PathUnescape(userid) + if err != nil { + o.mustRender(w, r, response.ErrRender(data.MetaServerError.StatusCode, err.Error())) + } // Really? a DELETE with form encoded body?!? // but it is not encoded as mime, so we cannot just call r.ParseForm() @@ -340,6 +349,10 @@ func (o Ocs) AddGroup(w http.ResponseWriter, r *http.Request) { // DeleteGroup deletes a group func (o Ocs) DeleteGroup(w http.ResponseWriter, r *http.Request) { groupid := chi.URLParam(r, "groupid") + groupid, err := url.PathUnescape(groupid) + if err != nil { + o.mustRender(w, r, response.ErrRender(data.MetaServerError.StatusCode, err.Error())) + } // ocs only knows about names so we have to look up the internal id group, err := o.fetchGroupByName(r.Context(), groupid) @@ -376,6 +389,10 @@ func (o Ocs) DeleteGroup(w http.ResponseWriter, r *http.Request) { func (o Ocs) GetGroupMembers(w http.ResponseWriter, r *http.Request) { groupid := chi.URLParam(r, "groupid") + groupid, err := url.PathUnescape(groupid) + if err != nil { + o.mustRender(w, r, response.ErrRender(data.MetaServerError.StatusCode, err.Error())) + } // ocs only knows about names so we have to look up the internal id group, err := o.fetchGroupByName(r.Context(), groupid) diff --git a/ocs/pkg/service/v0/users.go b/ocs/pkg/service/v0/users.go index fbad77da54..dd47b3d055 100644 --- a/ocs/pkg/service/v0/users.go +++ b/ocs/pkg/service/v0/users.go @@ -6,6 +6,7 @@ import ( "encoding/hex" "fmt" "net/http" + "net/url" "strconv" "strings" @@ -85,8 +86,11 @@ func (o Ocs) GetSelf(w http.ResponseWriter, r *http.Request) { // GetUser returns the user with the given userid func (o Ocs) GetUser(w http.ResponseWriter, r *http.Request) { userid := chi.URLParam(r, "userid") + userid, err := url.PathUnescape(userid) + if err != nil { + o.mustRender(w, r, response.ErrRender(data.MetaServerError.StatusCode, err.Error())) + } var account *accounts.Account - var err error switch { case userid == "": @@ -273,9 +277,12 @@ func (o Ocs) AddUser(w http.ResponseWriter, r *http.Request) { // EditUser creates a new user account func (o Ocs) EditUser(w http.ResponseWriter, r *http.Request) { userid := chi.URLParam(r, "userid") + userid, err := url.PathUnescape(userid) + if err != nil { + o.mustRender(w, r, response.ErrRender(data.MetaServerError.StatusCode, err.Error())) + } var account *accounts.Account - var err error switch o.config.AccountBackend { case "accounts": account, err = o.fetchAccountByUsername(r.Context(), userid) @@ -351,9 +358,12 @@ func (o Ocs) EditUser(w http.ResponseWriter, r *http.Request) { // DeleteUser deletes a user func (o Ocs) DeleteUser(w http.ResponseWriter, r *http.Request) { userid := chi.URLParam(r, "userid") + userid, err := url.PathUnescape(userid) + if err != nil { + o.mustRender(w, r, response.ErrRender(data.MetaServerError.StatusCode, err.Error())) + } var account *accounts.Account - var err error switch o.config.AccountBackend { case "accounts": account, err = o.fetchAccountByUsername(r.Context(), userid) @@ -508,9 +518,12 @@ func (o Ocs) mintTokenForUser(ctx context.Context, account *accounts.Account) (s // EnableUser enables a user func (o Ocs) EnableUser(w http.ResponseWriter, r *http.Request) { userid := chi.URLParam(r, "userid") + userid, err := url.PathUnescape(userid) + if err != nil { + o.mustRender(w, r, response.ErrRender(data.MetaServerError.StatusCode, err.Error())) + } var account *accounts.Account - var err error switch o.config.AccountBackend { case "accounts": account, err = o.fetchAccountByUsername(r.Context(), userid) @@ -559,9 +572,12 @@ func (o Ocs) EnableUser(w http.ResponseWriter, r *http.Request) { // DisableUser disables a user func (o Ocs) DisableUser(w http.ResponseWriter, r *http.Request) { userid := chi.URLParam(r, "userid") + userid, err := url.PathUnescape(userid) + if err != nil { + o.mustRender(w, r, response.ErrRender(data.MetaServerError.StatusCode, err.Error())) + } var account *accounts.Account - var err error switch o.config.AccountBackend { case "accounts": account, err = o.fetchAccountByUsername(r.Context(), userid) From a5accb2ce91b9330339c2ea9ff9684d2bb872077 Mon Sep 17 00:00:00 2001 From: Willy Kloucek Date: Mon, 6 Dec 2021 17:09:16 +0100 Subject: [PATCH 12/14] Revert "try without changed paths" This reverts commit 60d3962ee4566d01e1eff1b4f1f2fb69c129ea3a. --- .drone.star | 10 +++++----- proxy/pkg/config/config.go | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.drone.star b/.drone.star index 44f0363ef7..b912505fcc 100644 --- a/.drone.star +++ b/.drone.star @@ -1449,11 +1449,11 @@ def ocisServer(storage, accounts_hash_difficulty = 4, volumes = []): "OCIS_URL": "https://ocis-server:9200", "STORAGE_HOME_DRIVER": "%s" % (storage), "STORAGE_USERS_DRIVER": "%s" % (storage), - #"STORAGE_USERS_DRIVER_LOCAL_ROOT": "/srv/app/tmp/ocis/local/root", - #"STORAGE_USERS_DRIVER_OWNCLOUD_DATADIR": "/srv/app/tmp/ocis/owncloud/data", - #"STORAGE_USERS_DRIVER_OCIS_ROOT": "/srv/app/tmp/ocis/storage/users", - #"STORAGE_METADATA_DRIVER_OCIS_ROOT": "/srv/app/tmp/ocis/storage/metadata", - #"STORAGE_SHARING_USER_JSON_FILE": "/srv/app/tmp/ocis/shares.json", + "STORAGE_USERS_DRIVER_LOCAL_ROOT": "/srv/app/tmp/ocis/local/root", + "STORAGE_USERS_DRIVER_OWNCLOUD_DATADIR": "/srv/app/tmp/ocis/owncloud/data", + "STORAGE_USERS_DRIVER_OCIS_ROOT": "/srv/app/tmp/ocis/storage/users", + "STORAGE_METADATA_DRIVER_OCIS_ROOT": "/srv/app/tmp/ocis/storage/metadata", + "STORAGE_SHARING_USER_JSON_FILE": "/srv/app/tmp/ocis/shares.json", "PROXY_ENABLE_BASIC_AUTH": True, "WEB_UI_CONFIG": "/drone/src/tests/config/drone/ocis-config.json", "IDP_IDENTIFIER_REGISTRATION_CONF": "/drone/src/tests/config/drone/identifier-registration.yml", diff --git a/proxy/pkg/config/config.go b/proxy/pkg/config/config.go index 7a8b361487..bd21ee1e32 100644 --- a/proxy/pkg/config/config.go +++ b/proxy/pkg/config/config.go @@ -203,7 +203,7 @@ func New() *Config { func DefaultConfig() *Config { return &Config{ Debug: Debug{ - Addr: "127.0.0.1:9205", + Addr: "0.0.0.0:9205", Token: "", }, HTTP: HTTP{ From 34e755943a9fb053aa7e40347c41690322ef984f Mon Sep 17 00:00:00 2001 From: Willy Kloucek Date: Mon, 6 Dec 2021 17:16:59 +0100 Subject: [PATCH 13/14] add changelog --- .../fix-use-cs3apis-up-and-download-workflow-accounts.md | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 changelog/unreleased/fix-use-cs3apis-up-and-download-workflow-accounts.md diff --git a/changelog/unreleased/fix-use-cs3apis-up-and-download-workflow-accounts.md b/changelog/unreleased/fix-use-cs3apis-up-and-download-workflow-accounts.md new file mode 100644 index 0000000000..c24a553826 --- /dev/null +++ b/changelog/unreleased/fix-use-cs3apis-up-and-download-workflow-accounts.md @@ -0,0 +1,9 @@ +Bugfix: Use the CS3api up- and download workflow for the accounts service + +We've fixed the interaction of the accounts service with the metadata storage +after bypassing the InitiateUpload and InitiateDownload have been removed +from various storage drivers. The accounts service now uses the proper +CS3apis workflow for up- and downloads. + +https://github.com/owncloud/ocis/pull/2837 +https://github.com/cs3org/reva/pull/2309 From c338d322227ca10f7d021a50b3fa42bfc22919f3 Mon Sep 17 00:00:00 2001 From: Willy Kloucek Date: Tue, 7 Dec 2021 11:13:15 +0100 Subject: [PATCH 14/14] add failing tests to expected failures (https://github.com/owncloud/web/issues/4893) --- ...xpected-failures-webUI-on-OCIS-storage-ocisSmokeTest.md | 3 +++ .../acceptance/expected-failures-webUI-on-OCIS-storage.md | 7 +++++++ 2 files changed, 10 insertions(+) diff --git a/tests/acceptance/expected-failures-webUI-on-OCIS-storage-ocisSmokeTest.md b/tests/acceptance/expected-failures-webUI-on-OCIS-storage-ocisSmokeTest.md index 8bf9c61e4e..da270db559 100644 --- a/tests/acceptance/expected-failures-webUI-on-OCIS-storage-ocisSmokeTest.md +++ b/tests/acceptance/expected-failures-webUI-on-OCIS-storage-ocisSmokeTest.md @@ -10,6 +10,9 @@ Other free text and Markdown formatting can be used elsewhere in the document if Only the web scenarios tagged ocisSmokeTest are run by default in OCIS CI. This file lists the expected-failures of those ocisSmokeTest scenarios. +### [unexpected behavior when renaming files](https://github.com/owncloud/web/issues/4893) +- [webUIFilesCopy/copy.feature:36](https://github.com/owncloud/web/blob/master/tests/acceptance/features/webUIFilesCopy/copy.feature#L36) + ### [enable re-sharing is not possible](https://github.com/owncloud/ocis/issues/1743) - [webUISharingFilePermissionMultipleUsers/shareFileWithMultipleUsers.feature:67](https://github.com/owncloud/web/blob/master/tests/acceptance/features/webUISharingFilePermissionMultipleUsers/shareFileWithMultipleUsers.feature#L67) - [webUISharingFilePermissionMultipleUsers/shareFileWithMultipleUsers.feature:66](https://github.com/owncloud/web/blob/master/tests/acceptance/features/webUISharingFilePermissionMultipleUsers/shareFileWithMultipleUsers.feature#L66) diff --git a/tests/acceptance/expected-failures-webUI-on-OCIS-storage.md b/tests/acceptance/expected-failures-webUI-on-OCIS-storage.md index 75ea08d954..30ca76a42f 100644 --- a/tests/acceptance/expected-failures-webUI-on-OCIS-storage.md +++ b/tests/acceptance/expected-failures-webUI-on-OCIS-storage.md @@ -8,6 +8,13 @@ Level-3 headings should be used for the references to the relevant issues. Inclu Other free text and markdown formatting can be used elsewhere in the document if needed. But if you want to explain something about the issue, then please post that in the issue itself. +### [unexpected behavior when renaming files](https://github.com/owncloud/web/issues/4893) +- [webUIFilesCopy/copy.feature:36](https://github.com/owncloud/web/blob/master/tests/acceptance/features/webUIFilesCopy/copy.feature#L36) +- [webUIFilesCopy/copy.feature:68](https://github.com/owncloud/web/blob/master/tests/acceptance/features/webUIFilesCopy/copy.feature#L68) +- [webUIFilesCopy/copy.feature:69](https://github.com/owncloud/web/blob/master/tests/acceptance/features/webUIFilesCopy/copy.feature#L69) +- [webUIFilesCopy/copy.feature:70](https://github.com/owncloud/web/blob/master/tests/acceptance/features/webUIFilesCopy/copy.feature#L70) +- [webUIFilesCopy/copy.feature:71](https://github.com/owncloud/web/blob/master/tests/acceptance/features/webUIFilesCopy/copy.feature#L71) + ### [Media Viewer does not support mp3 files](https://github.com/owncloud/ocis/issues/1106) - [webUIPreview/imageMediaViewer.feature:84](https://github.com/owncloud/web/blob/master/tests/acceptance/features/webUIPreview/imageMediaViewer.feature#L84) - [webUIPreview/imageMediaViewer.feature:91](https://github.com/owncloud/web/blob/master/tests/acceptance/features/webUIPreview/imageMediaViewer.feature#L91)