Merge pull request #2837 from owncloud/update-reva-a042373b390

[full-ci] update reva, ignoring docker-arm

accounts/pkg/config/config.go

@@ -89,8 +89,6 @@ type Disk struct {
 // CS3 is the cs3 implementation of the storage.
 type CS3 struct {
 	ProviderAddr string `ocisConfig:"provider_addr"`
-	DataURL      string `ocisConfig:"data_url"`
-	DataPrefix   string `ocisConfig:"data_prefix"`
 	JWTSecret    string `ocisConfig:"jwt_secret"`
 }
 
@@ -185,8 +183,6 @@ func DefaultConfig() *Config {
 			},
 			CS3: CS3{
 				ProviderAddr: "localhost:9215",
-				DataURL:      "http://localhost:9216",
-				DataPrefix:   "data",
 				JWTSecret:    "Pive-Fumkiu4",
 			},
 		},

accounts/pkg/config/mappings.go

@@ -100,14 +100,6 @@ func structMappings(cfg *Config) []shared.EnvBinding {
 			EnvVars:     []string{"ACCOUNTS_STORAGE_CS3_PROVIDER_ADDR"},
 			Destination: &cfg.Repo.CS3.ProviderAddr,
 		},
-		{
-			EnvVars:     []string{"ACCOUNTS_STORAGE_CS3_DATA_URL"},
-			Destination: &cfg.Repo.CS3.DataURL,
-		},
-		{
-			EnvVars:     []string{"ACCOUNTS_STORAGE_CS3_DATA_PREFIX"},
-			Destination: &cfg.Repo.CS3.DataPrefix,
-		},
 		{
 			EnvVars:     []string{"OCIS_JWT_SECRET", "ACCOUNTS_STORAGE_CS3_JWT_SECRET"},
 			Destination: &cfg.Repo.CS3.JWTSecret,

accounts/pkg/service/v0/service.go

@@ -138,8 +138,6 @@ func configFromSvc(cfg *config.Config) (*idxcfg.Config, error) {
 			Backend: cfg.Repo.Backend,
 			CS3: idxcfg.CS3{
 				ProviderAddr: cfg.Repo.CS3.ProviderAddr,
-				DataURL:      cfg.Repo.CS3.DataURL,
-				DataPrefix:   cfg.Repo.CS3.DataPrefix,
 				JWTSecret:    cfg.Repo.CS3.JWTSecret,
 			},
 		}

accounts/pkg/storage/cs3.go

@@ -1,15 +1,10 @@
 package storage
 
 import (
-	"bytes"
 	"context"
 	"encoding/json"
-	"io"
-	"io/ioutil"
-	"net/http"
 	"path"
 	"path/filepath"
-	"strings"
 
 	"github.com/cs3org/reva/pkg/auth/scope"
 
@@ -23,15 +18,20 @@ import (
 	"github.com/owncloud/ocis/accounts/pkg/config"
 	"github.com/owncloud/ocis/accounts/pkg/proto/v0"
 	olog "github.com/owncloud/ocis/ocis-pkg/log"
+	metadatastorage "github.com/owncloud/ocis/ocis-pkg/metadata_storage"
 	"google.golang.org/grpc/metadata"
 )
 
+const (
+	storageMountPath = "/meta"
+)
+
 // CS3Repo provides a cs3 implementation of the Repo interface
 type CS3Repo struct {
 	cfg             *config.Config
 	tm              token.Manager
 	storageProvider provider.ProviderAPIClient
-	dataProvider    dataProviderClient // Used to create and download data via http, bypassing reva upload protocol
+	metadataStorage metadatastorage.MetadataStorage
 }
 
 // NewCS3Repo creates a new cs3 repo
@@ -49,26 +49,26 @@ func NewCS3Repo(cfg *config.Config) (Repo, error) {
 		return nil, err
 	}
 
+	ms, err := metadatastorage.NewMetadataStorage(cfg.Repo.CS3.ProviderAddr)
+	if err != nil {
+		return nil, err
+	}
+
 	return CS3Repo{
 		cfg:             cfg,
 		tm:              tokenManager,
 		storageProvider: client,
-		dataProvider: dataProviderClient{
-			client: http.Client{
-				Transport: http.DefaultTransport,
-			},
-		},
+		metadataStorage: ms,
 	}, nil
 }
 
 // WriteAccount writes an account via cs3 and modifies the provided account (e.g. with a generated id).
 func (r CS3Repo) WriteAccount(ctx context.Context, a *proto.Account) (err error) {
-	t, err := r.authenticate(ctx)
+	ctx, err = r.getAuthenticatedContext(ctx)
 	if err != nil {
 		return err
 	}
 
-	ctx = metadata.AppendToOutgoingContext(ctx, revactx.TokenHeader, t)
 	if err := r.makeRootDirIfNotExist(ctx, accountsFolder); err != nil {
 		return err
 	}
@@ -78,37 +78,31 @@ func (r CS3Repo) WriteAccount(ctx context.Context, a *proto.Account) (err error)
 		return err
 	}
 
-	resp, err := r.dataProvider.put(r.accountURL(a.Id), bytes.NewReader(by), t)
-	if err != nil {
-		return err
-	}
-	if err = resp.Body.Close(); err != nil {
-		return err
-	}
-	return nil
+	err = r.metadataStorage.SimpleUpload(ctx, r.accountURL(a.Id), by)
+	return err
+
 }
 
 // LoadAccount loads an account via cs3 by id and writes it to the provided account
 func (r CS3Repo) LoadAccount(ctx context.Context, id string, a *proto.Account) (err error) {
-	t, err := r.authenticate(ctx)
+	ctx, err = r.getAuthenticatedContext(ctx)
 	if err != nil {
 		return err
 	}
 
-	return r.loadAccount(id, t, a)
+	return r.loadAccount(ctx, id, a)
 }
 
 // LoadAccounts loads all the accounts from the cs3 api
 func (r CS3Repo) LoadAccounts(ctx context.Context, a *[]*proto.Account) (err error) {
-	t, err := r.authenticate(ctx)
+	ctx, err = r.getAuthenticatedContext(ctx)
 	if err != nil {
 		return err
 	}
 
-	ctx = metadata.AppendToOutgoingContext(ctx, revactx.TokenHeader, t)
 	res, err := r.storageProvider.ListContainer(ctx, &provider.ListContainerRequest{
 		Ref: &provider.Reference{
-			Path: path.Join("/meta", accountsFolder),
+			Path: path.Join(storageMountPath, accountsFolder),
 		},
 	})
 	if err != nil {
@@ -118,7 +112,7 @@ func (r CS3Repo) LoadAccounts(ctx context.Context, a *[]*proto.Account) (err err
 	log := olog.NewLogger(olog.Pretty(r.cfg.Log.Pretty), olog.Color(r.cfg.Log.Color), olog.Level(r.cfg.Log.Level))
 	for i := range res.Infos {
 		acc := &proto.Account{}
-		err := r.loadAccount(filepath.Base(res.Infos[i].Path), t, acc)
+		err := r.loadAccount(ctx, filepath.Base(res.Infos[i].Path), acc)
 		if err != nil {
 			log.Err(err).Msg("could not load account")
 			continue
@@ -128,38 +122,27 @@ func (r CS3Repo) LoadAccounts(ctx context.Context, a *[]*proto.Account) (err err
 	return nil
 }
 
-func (r CS3Repo) loadAccount(id string, t string, a *proto.Account) error {
-	resp, err := r.dataProvider.get(r.accountURL(id), t)
+func (r CS3Repo) loadAccount(ctx context.Context, id string, a *proto.Account) error {
+	account, err := r.metadataStorage.SimpleDownload(ctx, r.accountURL(id))
 	if err != nil {
+		if metadatastorage.IsNotFoundErr(err) {
+			return &notFoundErr{"account", id}
+		}
 		return err
 	}
-
-	if resp.StatusCode != http.StatusOK {
-		return &notFoundErr{"account", id}
-	}
-
-	b, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		return err
-	}
-	if err = resp.Body.Close(); err != nil {
-		return err
-	}
-	return json.Unmarshal(b, &a)
+	return json.Unmarshal(account, &a)
 }
 
 // DeleteAccount deletes an account via cs3 by id
 func (r CS3Repo) DeleteAccount(ctx context.Context, id string) (err error) {
-	t, err := r.authenticate(ctx)
+	ctx, err = r.getAuthenticatedContext(ctx)
 	if err != nil {
 		return err
 	}
 
-	ctx = metadata.AppendToOutgoingContext(ctx, revactx.TokenHeader, t)
-
 	resp, err := r.storageProvider.Delete(ctx, &provider.DeleteRequest{
 		Ref: &provider.Reference{
-			Path: path.Join("/meta", accountsFolder, id),
+			Path: path.Join(storageMountPath, accountsFolder, id),
 		},
 	})
 
@@ -177,12 +160,11 @@ func (r CS3Repo) DeleteAccount(ctx context.Context, id string) (err error) {
 
 // WriteGroup writes a group via cs3 and modifies the provided group (e.g. with a generated id).
 func (r CS3Repo) WriteGroup(ctx context.Context, g *proto.Group) (err error) {
-	t, err := r.authenticate(ctx)
+	ctx, err = r.getAuthenticatedContext(ctx)
 	if err != nil {
 		return err
 	}
 
-	ctx = metadata.AppendToOutgoingContext(ctx, revactx.TokenHeader, t)
 	if err := r.makeRootDirIfNotExist(ctx, groupsFolder); err != nil {
 		return err
 	}
@@ -192,37 +174,30 @@ func (r CS3Repo) WriteGroup(ctx context.Context, g *proto.Group) (err error) {
 		return err
 	}
 
-	resp, err := r.dataProvider.put(r.groupURL(g.Id), bytes.NewReader(by), t)
-	if err != nil {
-		return err
-	}
-	if err = resp.Body.Close(); err != nil {
-		return err
-	}
-	return nil
+	err = r.metadataStorage.SimpleUpload(ctx, r.groupURL(g.Id), by)
+	return err
 }
 
 // LoadGroup loads a group via cs3 by id and writes it to the provided group
 func (r CS3Repo) LoadGroup(ctx context.Context, id string, g *proto.Group) (err error) {
-	t, err := r.authenticate(ctx)
+	ctx, err = r.getAuthenticatedContext(ctx)
 	if err != nil {
 		return err
 	}
 
-	return r.loadGroup(id, t, g)
+	return r.loadGroup(ctx, id, g)
 }
 
 // LoadGroups loads all the groups from the cs3 api
 func (r CS3Repo) LoadGroups(ctx context.Context, g *[]*proto.Group) (err error) {
-	t, err := r.authenticate(ctx)
+	ctx, err = r.getAuthenticatedContext(ctx)
 	if err != nil {
 		return err
 	}
 
-	ctx = metadata.AppendToOutgoingContext(ctx, revactx.TokenHeader, t)
 	res, err := r.storageProvider.ListContainer(ctx, &provider.ListContainerRequest{
 		Ref: &provider.Reference{
-			Path: path.Join("/meta", groupsFolder),
+			Path: path.Join(storageMountPath, groupsFolder),
 		},
 	})
 	if err != nil {
@@ -232,7 +207,7 @@ func (r CS3Repo) LoadGroups(ctx context.Context, g *[]*proto.Group) (err error)
 	log := olog.NewLogger(olog.Pretty(r.cfg.Log.Pretty), olog.Color(r.cfg.Log.Color), olog.Level(r.cfg.Log.Level))
 	for i := range res.Infos {
 		grp := &proto.Group{}
-		err := r.loadGroup(filepath.Base(res.Infos[i].Path), t, grp)
+		err := r.loadGroup(ctx, filepath.Base(res.Infos[i].Path), grp)
 		if err != nil {
 			log.Err(err).Msg("could not load account")
 			continue
@@ -242,38 +217,27 @@ func (r CS3Repo) LoadGroups(ctx context.Context, g *[]*proto.Group) (err error)
 	return nil
 }
 
-func (r CS3Repo) loadGroup(id string, t string, g *proto.Group) error {
-	resp, err := r.dataProvider.get(r.groupURL(id), t)
+func (r CS3Repo) loadGroup(ctx context.Context, id string, g *proto.Group) error {
+	group, err := r.metadataStorage.SimpleDownload(ctx, r.groupURL(id))
 	if err != nil {
+		if metadatastorage.IsNotFoundErr(err) {
+			return &notFoundErr{"group", id}
+		}
 		return err
 	}
-
-	if resp.StatusCode == http.StatusNotFound {
-		return &notFoundErr{"group", id}
-	}
-
-	b, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		return err
-	}
-	if err = resp.Body.Close(); err != nil {
-		return err
-	}
-	return json.Unmarshal(b, &g)
+	return json.Unmarshal(group, &g)
 }
 
 // DeleteGroup deletes a group via cs3 by id
 func (r CS3Repo) DeleteGroup(ctx context.Context, id string) (err error) {
-	t, err := r.authenticate(ctx)
+	ctx, err = r.getAuthenticatedContext(ctx)
 	if err != nil {
 		return err
 	}
 
-	ctx = metadata.AppendToOutgoingContext(ctx, revactx.TokenHeader, t)
-
 	resp, err := r.storageProvider.Delete(ctx, &provider.DeleteRequest{
 		Ref: &provider.Reference{
-			Path: path.Join("/meta", groupsFolder, id),
+			Path: path.Join(storageMountPath, groupsFolder, id),
 		},
 	})
 
@@ -289,8 +253,13 @@ func (r CS3Repo) DeleteGroup(ctx context.Context, id string) (err error) {
 	return err
 }
 
-func (r CS3Repo) authenticate(ctx context.Context) (token string, err error) {
-	return AuthenticateCS3(ctx, r.cfg.ServiceUser, r.tm)
+func (r CS3Repo) getAuthenticatedContext(ctx context.Context) (context.Context, error) {
+	t, err := AuthenticateCS3(ctx, r.cfg.ServiceUser, r.tm)
+	if err != nil {
+		return nil, err
+	}
+	ctx = metadata.AppendToOutgoingContext(ctx, revactx.TokenHeader, t)
+	return ctx, nil
 }
 
 // AuthenticateCS3 mints an auth token for communicating with cs3 storage based on a service user from config
@@ -298,6 +267,7 @@ func AuthenticateCS3(ctx context.Context, su config.ServiceUser, tm token.Manage
 	u := &user.User{
 		Id: &user.UserId{
 			OpaqueId: su.UUID,
+			Type:     user.UserType_USER_TYPE_APPLICATION,
 		},
 		Groups:    []string{},
 		UidNumber: su.UID,
@@ -311,11 +281,11 @@ func AuthenticateCS3(ctx context.Context, su config.ServiceUser, tm token.Manage
 }
 
 func (r CS3Repo) accountURL(id string) string {
-	return singleJoiningSlash(r.cfg.Repo.CS3.DataURL, path.Join(r.cfg.Repo.CS3.DataPrefix, accountsFolder, id))
+	return path.Join(accountsFolder, id)
 }
 
 func (r CS3Repo) groupURL(id string) string {
-	return singleJoiningSlash(r.cfg.Repo.CS3.DataURL, path.Join(r.cfg.Repo.CS3.DataPrefix, groupsFolder, id))
+	return path.Join(groupsFolder, id)
 }
 
 func (r CS3Repo) makeRootDirIfNotExist(ctx context.Context, folder string) error {
@@ -325,7 +295,7 @@ func (r CS3Repo) makeRootDirIfNotExist(ctx context.Context, folder string) error
 // MakeDirIfNotExist will create a root node in the metadata storage. Requires an authenticated context.
 func MakeDirIfNotExist(ctx context.Context, sp provider.ProviderAPIClient, folder string) error {
 	var rootPathRef = &provider.Reference{
-		Path: path.Join("/meta", folder),
+		Path: path.Join(storageMountPath, folder),
 	}
 
 	resp, err := sp.Stat(ctx, &provider.StatRequest{
@@ -348,40 +318,3 @@ func MakeDirIfNotExist(ctx context.Context, sp provider.ProviderAPIClient, folde
 
 	return nil
 }
-
-// TODO: this is copied from proxy. Find a better solution or move it to ocis-pkg
-func singleJoiningSlash(a, b string) string {
-	aslash := strings.HasSuffix(a, "/")
-	bslash := strings.HasPrefix(b, "/")
-	switch {
-	case aslash && bslash:
-		return a + b[1:]
-	case !aslash && !bslash:
-		return a + "/" + b
-	}
-	return a + b
-}
-
-type dataProviderClient struct {
-	client http.Client
-}
-
-func (d dataProviderClient) put(url string, body io.Reader, token string) (*http.Response, error) {
-	req, err := http.NewRequest(http.MethodPut, url, body)
-	if err != nil {
-		return nil, err
-	}
-
-	req.Header.Add(revactx.TokenHeader, token)
-	return d.client.Do(req)
-}
-
-func (d dataProviderClient) get(url string, token string) (*http.Response, error) {
-	req, err := http.NewRequest(http.MethodGet, url, nil)
-	if err != nil {
-		return nil, err
-	}
-
-	req.Header.Add(revactx.TokenHeader, token)
-	return d.client.Do(req)
-}

accounts/pkg/storage/cs3_test.go

@@ -17,8 +17,6 @@ package storage
 //	Repo: config.Repo{
 //		CS3:  config.CS3{
 //			ProviderAddr: "0.0.0.0:9215",
-//			DataURL:    "http://localhost:9216",
-//			DataPrefix:   "data",
 //		},
 //	},
 //}

accounts/pkg/storage/errors.go

@@ -8,7 +8,7 @@ type notFoundErr struct {
 	typ, id string
 }
 
-func (e *notFoundErr) Error() string {
+func (e notFoundErr) Error() string {
 	return fmt.Sprintf("%s with id %s not found", e.typ, e.id)
 }
 

changelog/unreleased/fix-use-cs3apis-up-and-download-workflow-accounts.md

@@ -0,0 +1,9 @@
+Bugfix: Use the CS3api up- and download workflow for the accounts service
+
+We've fixed the interaction of the accounts service with the metadata storage
+after bypassing the InitiateUpload and InitiateDownload have been removed
+from various storage drivers. The accounts service now uses the proper
+CS3apis workflow for up- and downloads.
+
+https://github.com/owncloud/ocis/pull/2837
+https://github.com/cs3org/reva/pull/2309

changelog/unreleased/update-reva.md

@@ -7,3 +7,4 @@ This update includes:
 
 
 https://github.com/owncloud/ocis/pull/2835
+https://github.com/owncloud/ocis/pull/2837

go.mod

@@ -19,7 +19,7 @@ require (
 	github.com/blevesearch/bleve/v2 v2.2.2
 	github.com/coreos/go-oidc/v3 v3.1.0
 	github.com/cs3org/go-cs3apis v0.0.0-20211104090126-8e972dca8304
-	github.com/cs3org/reva v1.16.1-0.20211129095857-79fb2995dab9
+	github.com/cs3org/reva v1.16.1-0.20211203225713-939768a1af06
 	github.com/disintegration/imaging v1.6.2
 	github.com/glauth/glauth/v2 v2.0.0-20211021011345-ef3151c28733
 	github.com/go-chi/chi/v5 v5.0.7

go.sum

@@ -299,8 +299,8 @@ github.com/crewjam/saml v0.4.5/go.mod h1:qCJQpUtZte9R1ZjUBcW8qtCNlinbO363ooNl02S
 github.com/cs3org/cato v0.0.0-20200828125504-e418fc54dd5e/go.mod h1:XJEZ3/EQuI3BXTp/6DUzFr850vlxq11I6satRtz0YQ4=
 github.com/cs3org/go-cs3apis v0.0.0-20211104090126-8e972dca8304 h1:e/nIPR518vyvrulo9goAZTtYD6gFfu/2/9MDe6mTGcw=
 github.com/cs3org/go-cs3apis v0.0.0-20211104090126-8e972dca8304/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY=
-github.com/cs3org/reva v1.16.1-0.20211129095857-79fb2995dab9 h1:fgnLZB/tZCH4Kl2+ZBOpRd/dZLVF7VCrSFaL1FvZgwE=
-github.com/cs3org/reva v1.16.1-0.20211129095857-79fb2995dab9/go.mod h1:3n/zVKsKTCL10Mwn2Nhtvn50gP5mA+933lQ2IYNAJso=
+github.com/cs3org/reva v1.16.1-0.20211203225713-939768a1af06 h1:NP+Zvli7+9USaDss/+Ywk4KJ0H7n82UHZiU4V+x25I0=
+github.com/cs3org/reva v1.16.1-0.20211203225713-939768a1af06/go.mod h1:3n/zVKsKTCL10Mwn2Nhtvn50gP5mA+933lQ2IYNAJso=
 github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8 h1:Z9lwXumT5ACSmJ7WGnFl+OMLLjpz5uR2fyz7dC255FI=
 github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8/go.mod h1:4abs/jPXcmJzYoYGF91JF9Uq9s/KL5n1jvFDix8KcqY=
 github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4=

graph/pkg/service/v0/groups.go

@@ -3,6 +3,7 @@ package svc
 import (
 	"errors"
 	"net/http"
+	"net/url"
 
 	"github.com/owncloud/ocis/graph/pkg/service/v0/errorcode"
 
@@ -30,6 +31,11 @@ func (g Graph) GetGroups(w http.ResponseWriter, r *http.Request) {
 // GetGroup implements the Service interface.
 func (g Graph) GetGroup(w http.ResponseWriter, r *http.Request) {
 	groupID := chi.URLParam(r, "groupID")
+	groupID, err := url.PathUnescape(groupID)
+	if err != nil {
+		errorcode.InvalidRequest.Render(w, r, http.StatusBadRequest, "unescaping group id failed")
+	}
+
 	if groupID == "" {
 		errorcode.InvalidRequest.Render(w, r, http.StatusBadRequest, "missing group id")
 		return

graph/pkg/service/v0/users.go

@@ -3,6 +3,7 @@ package svc
 import (
 	"errors"
 	"net/http"
+	"net/url"
 
 	revactx "github.com/cs3org/reva/pkg/ctx"
 	"github.com/go-chi/chi/v5"
@@ -49,6 +50,11 @@ func (g Graph) GetUsers(w http.ResponseWriter, r *http.Request) {
 // GetUser implements the Service interface.
 func (g Graph) GetUser(w http.ResponseWriter, r *http.Request) {
 	userID := chi.URLParam(r, "userID")
+	userID, err := url.PathUnescape(userID)
+	if err != nil {
+		errorcode.InvalidRequest.Render(w, r, http.StatusBadRequest, "unescaping user id failed")
+	}
+
 	if userID == "" {
 		errorcode.InvalidRequest.Render(w, r, http.StatusBadRequest, "missing user id")
 		return

ocis-pkg/indexer/index/cs3/autoincrement.go

@@ -2,9 +2,6 @@ package cs3
 
 import (
 	"context"
-	"fmt"
-	"io/ioutil"
-	"net/http"
 	"os"
 	"path"
 	"path/filepath"
@@ -27,6 +24,7 @@ import (
 	"github.com/owncloud/ocis/ocis-pkg/indexer/index"
 	"github.com/owncloud/ocis/ocis-pkg/indexer/option"
 	"github.com/owncloud/ocis/ocis-pkg/indexer/registry"
+	metadatastorage "github.com/owncloud/ocis/ocis-pkg/metadata_storage"
 )
 
 // Autoincrement are fields for an index of type autoincrement.
@@ -39,7 +37,7 @@ type Autoincrement struct {
 
 	tokenManager    token.Manager
 	storageProvider provider.ProviderAPIClient
-	dataProvider    dataProviderClient // Used to create and download data via http, bypassing reva upload protocol
+	metadataStorage *metadatastorage.MetadataStorage
 
 	cs3conf *Config
 	bound   *option.Bound
@@ -65,17 +63,9 @@ func NewAutoincrementIndex(o ...option.Option) index.Index {
 		indexRootDir: path.Join(path.Join(opts.DataDir, "index.cs3"), strings.Join([]string{"autoincrement", opts.TypeName, opts.IndexBy}, ".")),
 		cs3conf: &Config{
 			ProviderAddr: opts.ProviderAddr,
-			DataURL:      opts.DataURL,
-			DataPrefix:   opts.DataPrefix,
 			JWTSecret:    opts.JWTSecret,
 			ServiceUser:  opts.ServiceUser,
 		},
-		dataProvider: dataProviderClient{
-			baseURL: singleJoiningSlash(opts.DataURL, opts.DataPrefix),
-			client: http.Client{
-				Transport: http.DefaultTransport,
-			},
-		},
 	}
 
 	return u
@@ -86,30 +76,28 @@ func (idx *Autoincrement) Init() error {
 	tokenManager, err := jwt.New(map[string]interface{}{
 		"secret": idx.cs3conf.JWTSecret,
 	})
-
 	if err != nil {
 		return err
 	}
-
 	idx.tokenManager = tokenManager
 
 	client, err := pool.GetStorageProviderServiceClient(idx.cs3conf.ProviderAddr)
 	if err != nil {
 		return err
 	}
-
 	idx.storageProvider = client
 
-	ctx, err := idx.getAuthenticatedContext(context.Background())
+	m, err := metadatastorage.NewMetadataStorage(idx.cs3conf.ProviderAddr)
 	if err != nil {
 		return err
 	}
+	idx.metadataStorage = &m
 
-	if err := idx.makeDirIfNotExists(ctx, idx.indexBaseDir); err != nil {
+	if err := idx.makeDirIfNotExists(idx.indexBaseDir); err != nil {
 		return err
 	}
 
-	if err := idx.makeDirIfNotExists(ctx, idx.indexRootDir); err != nil {
+	if err := idx.makeDirIfNotExists(idx.indexRootDir); err != nil {
 		return err
 	}
 
@@ -263,7 +251,7 @@ func (idx *Autoincrement) FilesDir() string {
 }
 
 func (idx *Autoincrement) createSymlink(oldname, newname string) error {
-	t, err := idx.authenticate(context.TODO())
+	ctx, err := idx.getAuthenticatedContext(context.Background())
 	if err != nil {
 		return err
 	}
@@ -272,52 +260,37 @@ func (idx *Autoincrement) createSymlink(oldname, newname string) error {
 		return os.ErrExist
 	}
 
-	resp, err := idx.dataProvider.put(newname, strings.NewReader(oldname), t)
+	err = idx.metadataStorage.SimpleUpload(ctx, newname, []byte(oldname))
 	if err != nil {
 		return err
 	}
-	if err = resp.Body.Close(); err != nil {
-		return err
-	}
-
 	return nil
 }
 
 func (idx *Autoincrement) resolveSymlink(name string) (string, error) {
-	t, err := idx.authenticate(context.TODO())
+	ctx, err := idx.getAuthenticatedContext(context.Background())
 	if err != nil {
 		return "", err
 	}
 
-	resp, err := idx.dataProvider.get(name, t)
+	b, err := idx.metadataStorage.SimpleDownload(ctx, name)
 	if err != nil {
-		return "", err
-	}
-
-	if resp.StatusCode != http.StatusOK {
-		if resp.StatusCode == http.StatusNotFound {
+		if metadatastorage.IsNotFoundErr(err) {
 			return "", os.ErrNotExist
 		}
-
-		return "", fmt.Errorf("could not resolve symlink %s, got status %v", name, resp.StatusCode)
-	}
-
-	b, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		return "", err
-	}
-	if err = resp.Body.Close(); err != nil {
 		return "", err
 	}
+
 	return string(b), err
 }
 
-func (idx *Autoincrement) makeDirIfNotExists(ctx context.Context, folder string) error {
-	return storage.MakeDirIfNotExist(ctx, idx.storageProvider, folder)
-}
+func (idx *Autoincrement) makeDirIfNotExists(folder string) error {
+	ctx, err := idx.getAuthenticatedContext(context.Background())
+	if err != nil {
+		return err
+	}
 
-func (idx *Autoincrement) authenticate(ctx context.Context) (token string, err error) {
-	return storage.AuthenticateCS3(ctx, idx.cs3conf.ServiceUser, idx.tokenManager)
+	return storage.MakeDirIfNotExist(ctx, idx.storageProvider, folder)
 }
 
 func (idx *Autoincrement) next() (int, error) {
@@ -328,7 +301,7 @@ func (idx *Autoincrement) next() (int, error) {
 
 	res, err := idx.storageProvider.ListContainer(ctx, &provider.ListContainerRequest{
 		Ref: &provider.Reference{
-			Path: path.Join("/meta", idx.indexRootDir),
+			Path: path.Join("/meta", idx.indexRootDir), //TODO:
 		},
 	})
 
@@ -360,7 +333,7 @@ func (idx *Autoincrement) next() (int, error) {
 }
 
 func (idx *Autoincrement) getAuthenticatedContext(ctx context.Context) (context.Context, error) {
-	t, err := idx.authenticate(ctx)
+	t, err := storage.AuthenticateCS3(ctx, idx.cs3conf.ServiceUser, idx.tokenManager)
 	if err != nil {
 		return nil, err
 	}

ocis-pkg/indexer/index/cs3/config.go

@@ -0,0 +1,12 @@
+package cs3
+
+import (
+	acccfg "github.com/owncloud/ocis/accounts/pkg/config"
+)
+
+// Config represents cs3conf. Should be deprecated in favor of config.Config.
+type Config struct {
+	ProviderAddr string
+	JWTSecret    string
+	ServiceUser  acccfg.ServiceUser
+}

ocis-pkg/indexer/index/cs3/data_provider_client.go

@@ -1,45 +0,0 @@
-package cs3
-
-import (
-	"io"
-	"net/http"
-	"strings"
-)
-
-type dataProviderClient struct {
-	client  http.Client
-	baseURL string
-}
-
-func (d dataProviderClient) put(url string, body io.Reader, token string) (*http.Response, error) {
-	req, err := http.NewRequest(http.MethodPut, singleJoiningSlash(d.baseURL, url), body)
-	if err != nil {
-		return nil, err
-	}
-
-	req.Header.Add("x-access-token", token)
-	return d.client.Do(req)
-}
-
-func (d dataProviderClient) get(url string, token string) (*http.Response, error) {
-	req, err := http.NewRequest(http.MethodGet, singleJoiningSlash(d.baseURL, url), nil)
-	if err != nil {
-		return nil, err
-	}
-
-	req.Header.Add("x-access-token", token)
-	return d.client.Do(req)
-}
-
-// TODO: this is copied from proxy. Find a better solution or move it to ocis-pkg
-func singleJoiningSlash(a, b string) string {
-	aslash := strings.HasSuffix(a, "/")
-	bslash := strings.HasPrefix(b, "/")
-	switch {
-	case aslash && bslash:
-		return a + b[1:]
-	case !aslash && !bslash:
-		return a + "/" + b
-	}
-	return a + b
-}

ocis-pkg/indexer/index/cs3/non_unique.go

@@ -2,9 +2,6 @@ package cs3
 
 import (
 	"context"
-	"fmt"
-	"io/ioutil"
-	"net/http"
 	"os"
 	"path"
 	"path/filepath"
@@ -22,6 +19,7 @@ import (
 	"github.com/owncloud/ocis/ocis-pkg/indexer/index"
 	"github.com/owncloud/ocis/ocis-pkg/indexer/option"
 	"github.com/owncloud/ocis/ocis-pkg/indexer/registry"
+	metadatastorage "github.com/owncloud/ocis/ocis-pkg/metadata_storage"
 	"google.golang.org/grpc/metadata"
 )
 
@@ -40,7 +38,7 @@ type NonUnique struct {
 
 	tokenManager    token.Manager
 	storageProvider provider.ProviderAPIClient
-	dataProvider    dataProviderClient // Used to create and download data via http, bypassing reva upload protocol
+	metadataStorage *metadatastorage.MetadataStorage
 
 	cs3conf *Config
 }
@@ -69,17 +67,9 @@ func NewNonUniqueIndexWithOptions(o ...option.Option) index.Index {
 		indexRootDir:    path.Join(path.Join(opts.DataDir, "index.cs3"), strings.Join([]string{"non_unique", opts.TypeName, opts.IndexBy}, ".")),
 		cs3conf: &Config{
 			ProviderAddr: opts.ProviderAddr,
-			DataURL:      opts.DataURL,
-			DataPrefix:   opts.DataPrefix,
 			JWTSecret:    opts.JWTSecret,
 			ServiceUser:  opts.ServiceUser,
 		},
-		dataProvider: dataProviderClient{
-			baseURL: singleJoiningSlash(opts.DataURL, opts.DataPrefix),
-			client: http.Client{
-				Transport: http.DefaultTransport,
-			},
-		},
 	}
 }
 
@@ -88,32 +78,28 @@ func (idx *NonUnique) Init() error {
 	tokenManager, err := jwt.New(map[string]interface{}{
 		"secret": idx.cs3conf.JWTSecret,
 	})
-
 	if err != nil {
 		return err
 	}
-
 	idx.tokenManager = tokenManager
 
 	client, err := pool.GetStorageProviderServiceClient(idx.cs3conf.ProviderAddr)
 	if err != nil {
 		return err
 	}
-
 	idx.storageProvider = client
 
-	ctx := context.Background()
-	tk, err := idx.authenticate(ctx)
+	m, err := metadatastorage.NewMetadataStorage(idx.cs3conf.ProviderAddr)
 	if err != nil {
 		return err
 	}
-	ctx = metadata.AppendToOutgoingContext(ctx, revactx.TokenHeader, tk)
+	idx.metadataStorage = &m
 
-	if err := idx.makeDirIfNotExists(ctx, idx.indexBaseDir); err != nil {
+	if err := idx.makeDirIfNotExists(idx.indexBaseDir); err != nil {
 		return err
 	}
 
-	if err := idx.makeDirIfNotExists(ctx, idx.indexRootDir); err != nil {
+	if err := idx.makeDirIfNotExists(idx.indexRootDir); err != nil {
 		return err
 	}
 
@@ -156,13 +142,9 @@ func (idx *NonUnique) Add(id, v string) (string, error) {
 	if idx.caseInsensitive {
 		v = strings.ToLower(v)
 	}
-	ctx, err := idx.getAuthenticatedContext(context.Background())
-	if err != nil {
-		return "", err
-	}
 
 	newName := path.Join(idx.indexRootDir, v)
-	if err := idx.makeDirIfNotExists(ctx, newName); err != nil {
+	if err := idx.makeDirIfNotExists(newName); err != nil {
 		return "", err
 	}
 
@@ -316,12 +298,16 @@ func (idx *NonUnique) FilesDir() string {
 	return idx.filesDir
 }
 
-func (idx *NonUnique) makeDirIfNotExists(ctx context.Context, folder string) error {
+func (idx *NonUnique) makeDirIfNotExists(folder string) error {
+	ctx, err := idx.getAuthenticatedContext(context.Background())
+	if err != nil {
+		return err
+	}
 	return storage.MakeDirIfNotExist(ctx, idx.storageProvider, folder)
 }
 
 func (idx *NonUnique) createSymlink(oldname, newname string) error {
-	t, err := idx.authenticate(context.TODO())
+	ctx, err := idx.getAuthenticatedContext(context.Background())
 	if err != nil {
 		return err
 	}
@@ -330,48 +316,32 @@ func (idx *NonUnique) createSymlink(oldname, newname string) error {
 		return os.ErrExist
 	}
 
-	resp, err := idx.dataProvider.put(newname, strings.NewReader(oldname), t)
+	err = idx.metadataStorage.SimpleUpload(ctx, newname, []byte(oldname))
 	if err != nil {
 		return err
 	}
-	if err = resp.Body.Close(); err != nil {
-		return err
-	}
-
 	return nil
 }
 
 func (idx *NonUnique) resolveSymlink(name string) (string, error) {
-	t, err := idx.authenticate(context.TODO())
+	ctx, err := idx.getAuthenticatedContext(context.Background())
 	if err != nil {
 		return "", err
 	}
 
-	resp, err := idx.dataProvider.get(name, t)
+	b, err := idx.metadataStorage.SimpleDownload(ctx, name)
 	if err != nil {
-		return "", err
-	}
-
-	if resp.StatusCode != http.StatusOK {
-		if resp.StatusCode == http.StatusNotFound {
+		if metadatastorage.IsNotFoundErr(err) {
 			return "", os.ErrNotExist
 		}
-
-		return "", fmt.Errorf("could not resolve symlink %s, got status %v", name, resp.StatusCode)
-	}
-
-	b, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		return "", err
-	}
-	if err = resp.Body.Close(); err != nil {
 		return "", err
 	}
+
 	return string(b), err
 }
 
 func (idx *NonUnique) getAuthenticatedContext(ctx context.Context) (context.Context, error) {
-	t, err := idx.authenticate(ctx)
+	t, err := storage.AuthenticateCS3(ctx, idx.cs3conf.ServiceUser, idx.tokenManager)
 	if err != nil {
 		return nil, err
 	}
@@ -388,7 +358,3 @@ func (idx *NonUnique) Delete() error {
 
 	return deleteIndexRoot(ctx, idx.storageProvider, idx.indexRootDir)
 }
-
-func (idx *NonUnique) authenticate(ctx context.Context) (token string, err error) {
-	return storage.AuthenticateCS3(ctx, idx.cs3conf.ServiceUser, idx.tokenManager)
-}

ocis-pkg/indexer/index/cs3/unique.go

@@ -2,9 +2,6 @@ package cs3
 
 import (
 	"context"
-	"fmt"
-	"io/ioutil"
-	"net/http"
 	"os"
 	"path"
 	"path/filepath"
@@ -12,8 +9,6 @@ import (
 
 	"github.com/owncloud/ocis/accounts/pkg/storage"
 
-	acccfg "github.com/owncloud/ocis/accounts/pkg/config"
-
 	v1beta11 "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1"
 	provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
 	revactx "github.com/cs3org/reva/pkg/ctx"
@@ -24,6 +19,7 @@ import (
 	"github.com/owncloud/ocis/ocis-pkg/indexer/index"
 	"github.com/owncloud/ocis/ocis-pkg/indexer/option"
 	"github.com/owncloud/ocis/ocis-pkg/indexer/registry"
+	metadatastorage "github.com/owncloud/ocis/ocis-pkg/metadata_storage"
 	"google.golang.org/grpc/metadata"
 )
 
@@ -38,20 +34,11 @@ type Unique struct {
 
 	tokenManager    token.Manager
 	storageProvider provider.ProviderAPIClient
-	dataProvider    dataProviderClient // Used to create and download data via http, bypassing reva upload protocol
+	metadataStorage *metadatastorage.MetadataStorage
 
 	cs3conf *Config
 }
 
-// Config represents cs3conf. Should be deprecated in favor of config.Config.
-type Config struct {
-	ProviderAddr string
-	DataURL      string
-	DataPrefix   string
-	JWTSecret    string
-	ServiceUser  acccfg.ServiceUser
-}
-
 func init() {
 	registry.IndexConstructorRegistry["cs3"]["unique"] = NewUniqueIndexWithOptions
 }
@@ -73,17 +60,9 @@ func NewUniqueIndexWithOptions(o ...option.Option) index.Index {
 		indexRootDir:    path.Join(path.Join(opts.DataDir, "index.cs3"), strings.Join([]string{"unique", opts.TypeName, opts.IndexBy}, ".")),
 		cs3conf: &Config{
 			ProviderAddr: opts.ProviderAddr,
-			DataURL:      opts.DataURL,
-			DataPrefix:   opts.DataPrefix,
 			JWTSecret:    opts.JWTSecret,
 			ServiceUser:  opts.ServiceUser,
 		},
-		dataProvider: dataProviderClient{
-			baseURL: singleJoiningSlash(opts.DataURL, opts.DataPrefix),
-			client: http.Client{
-				Transport: http.DefaultTransport,
-			},
-		},
 	}
 
 	return u
@@ -94,32 +73,28 @@ func (idx *Unique) Init() error {
 	tokenManager, err := jwt.New(map[string]interface{}{
 		"secret": idx.cs3conf.JWTSecret,
 	})
-
 	if err != nil {
 		return err
 	}
-
 	idx.tokenManager = tokenManager
 
 	client, err := pool.GetStorageProviderServiceClient(idx.cs3conf.ProviderAddr)
 	if err != nil {
 		return err
 	}
-
 	idx.storageProvider = client
 
-	ctx := context.Background()
-	tk, err := idx.authenticate(ctx)
+	m, err := metadatastorage.NewMetadataStorage(idx.cs3conf.ProviderAddr)
 	if err != nil {
 		return err
 	}
-	ctx = metadata.AppendToOutgoingContext(ctx, revactx.TokenHeader, tk)
+	idx.metadataStorage = &m
 
-	if err := idx.makeDirIfNotExists(ctx, idx.indexBaseDir); err != nil {
+	if err := idx.makeDirIfNotExists(idx.indexBaseDir); err != nil {
 		return err
 	}
 
-	if err := idx.makeDirIfNotExists(ctx, idx.indexRootDir); err != nil {
+	if err := idx.makeDirIfNotExists(idx.indexRootDir); err != nil {
 		return err
 	}
 
@@ -182,14 +157,12 @@ func (idx *Unique) Remove(id string, v string) error {
 		return err
 	}
 
-	ctx := context.Background()
-	t, err := idx.authenticate(ctx)
+	ctx, err := idx.getAuthenticatedContext(context.Background())
 	if err != nil {
 		return err
 	}
 
 	deletePath := path.Join("/meta", idx.indexRootDir, v)
-	ctx = metadata.AppendToOutgoingContext(ctx, revactx.TokenHeader, t)
 	resp, err := idx.storageProvider.Delete(ctx, &provider.DeleteRequest{
 		Ref: &provider.Reference{
 			Path: deletePath,
@@ -232,13 +205,11 @@ func (idx *Unique) Search(pattern string) ([]string, error) {
 		pattern = strings.ToLower(pattern)
 	}
 
-	ctx := context.Background()
-	t, err := idx.authenticate(ctx)
+	ctx, err := idx.getAuthenticatedContext(context.Background())
 	if err != nil {
 		return nil, err
 	}
 
-	ctx = metadata.AppendToOutgoingContext(ctx, revactx.TokenHeader, t)
 	res, err := idx.storageProvider.ListContainer(ctx, &provider.ListContainerRequest{
 		Ref: &provider.Reference{
 			Path: path.Join("/meta", idx.indexRootDir),
@@ -289,7 +260,7 @@ func (idx *Unique) FilesDir() string {
 }
 
 func (idx *Unique) createSymlink(oldname, newname string) error {
-	t, err := idx.authenticate(context.TODO())
+	ctx, err := idx.getAuthenticatedContext(context.Background())
 	if err != nil {
 		return err
 	}
@@ -298,56 +269,41 @@ func (idx *Unique) createSymlink(oldname, newname string) error {
 		return os.ErrExist
 	}
 
-	resp, err := idx.dataProvider.put(newname, strings.NewReader(oldname), t)
+	err = idx.metadataStorage.SimpleUpload(ctx, newname, []byte(oldname))
 	if err != nil {
 		return err
 	}
-	if err = resp.Body.Close(); err != nil {
-		return err
-	}
 
 	return nil
 }
 
 func (idx *Unique) resolveSymlink(name string) (string, error) {
-	t, err := idx.authenticate(context.TODO())
+	ctx, err := idx.getAuthenticatedContext(context.Background())
 	if err != nil {
 		return "", err
 	}
 
-	resp, err := idx.dataProvider.get(name, t)
+	b, err := idx.metadataStorage.SimpleDownload(ctx, name)
 	if err != nil {
-		return "", err
-	}
-
-	if resp.StatusCode != http.StatusOK {
-		if resp.StatusCode == http.StatusNotFound {
+		if metadatastorage.IsNotFoundErr(err) {
 			return "", os.ErrNotExist
 		}
-
-		return "", fmt.Errorf("could not resolve symlink %s, got status %v", name, resp.StatusCode)
-	}
-
-	b, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		return "", err
-	}
-	if err = resp.Body.Close(); err != nil {
 		return "", err
 	}
+
 	return string(b), err
 }
 
-func (idx *Unique) makeDirIfNotExists(ctx context.Context, folder string) error {
+func (idx *Unique) makeDirIfNotExists(folder string) error {
+	ctx, err := idx.getAuthenticatedContext(context.Background())
+	if err != nil {
+		return err
+	}
 	return storage.MakeDirIfNotExist(ctx, idx.storageProvider, folder)
 }
 
-func (idx *Unique) authenticate(ctx context.Context) (token string, err error) {
-	return storage.AuthenticateCS3(ctx, idx.cs3conf.ServiceUser, idx.tokenManager)
-}
-
 func (idx *Unique) getAuthenticatedContext(ctx context.Context) (context.Context, error) {
-	t, err := idx.authenticate(ctx)
+	t, err := storage.AuthenticateCS3(ctx, idx.cs3conf.ServiceUser, idx.tokenManager)
 	if err != nil {
 		return nil, err
 	}

ocis-pkg/metadata_storage/errors.go

@@ -0,0 +1,17 @@
+package metadataStorage
+
+import "fmt"
+
+type notFoundErr struct {
+	typ, id string
+}
+
+func (e notFoundErr) Error() string {
+	return fmt.Sprintf("%s with id %s not found", e.typ, e.id)
+}
+
+// IsNotFoundErr can be returned by repo Load and Delete operations
+func IsNotFoundErr(e error) bool {
+	_, ok := e.(*notFoundErr)
+	return ok
+}

ocis-pkg/metadata_storage/metadata_storage.go

@@ -0,0 +1,133 @@
+package metadataStorage
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"io/ioutil"
+	"net/http"
+	"path"
+
+	provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
+	revactx "github.com/cs3org/reva/pkg/ctx"
+	"github.com/cs3org/reva/pkg/rgrpc/todo/pool"
+	"google.golang.org/grpc/metadata"
+)
+
+const (
+	storageMountPath = "/meta"
+)
+
+func NewMetadataStorage(providerAddr string) (s MetadataStorage, err error) {
+
+	p, err := pool.GetStorageProviderServiceClient(providerAddr)
+	if err != nil {
+		return MetadataStorage{}, err
+	}
+
+	c := http.DefaultClient
+
+	return MetadataStorage{
+		storageProvider:   p,
+		dataGatewayClient: c,
+	}, nil
+}
+
+type MetadataStorage struct {
+	storageProvider   provider.ProviderAPIClient
+	dataGatewayClient *http.Client
+}
+
+func (r MetadataStorage) SimpleUpload(ctx context.Context, uploadpath string, content []byte) error {
+
+	ref := provider.InitiateFileUploadRequest{
+		Ref: &provider.Reference{
+			Path: path.Join(storageMountPath, uploadpath),
+		},
+	}
+
+	res, err := r.storageProvider.InitiateFileUpload(ctx, &ref)
+	if err != nil {
+		return err
+	}
+
+	var endpoint string
+
+	for _, proto := range res.GetProtocols() {
+		if proto.Protocol == "simple" {
+			endpoint = proto.GetUploadEndpoint()
+			break
+		}
+	}
+	if endpoint == "" {
+		return errors.New("metadata storage doesn't support the simple upload protocol")
+	}
+
+	req, err := http.NewRequest(http.MethodPut, endpoint, bytes.NewReader(content))
+	if err != nil {
+		return err
+	}
+
+	md, _ := metadata.FromOutgoingContext(ctx)
+	req.Header.Add(revactx.TokenHeader, md.Get(revactx.TokenHeader)[0])
+	resp, err := r.dataGatewayClient.Do(req)
+	if err != nil {
+		return err
+	}
+	if err = resp.Body.Close(); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (r MetadataStorage) SimpleDownload(ctx context.Context, downloadpath string) (content []byte, err error) {
+	ref := provider.InitiateFileDownloadRequest{
+		Ref: &provider.Reference{
+			Path: path.Join(storageMountPath, downloadpath),
+		},
+	}
+
+	res, err := r.storageProvider.InitiateFileDownload(ctx, &ref)
+	if err != nil {
+		return []byte{}, err
+	}
+
+	var endpoint string
+
+	for _, proto := range res.GetProtocols() {
+		if proto.Protocol == "simple" {
+			endpoint = proto.GetDownloadEndpoint()
+			break
+		}
+	}
+	if endpoint == "" {
+		return []byte{}, errors.New("metadata storage doesn't support the simple download protocol")
+	}
+
+	req, err := http.NewRequest(http.MethodGet, endpoint, nil)
+	if err != nil {
+		return []byte{}, err
+	}
+
+	md, _ := metadata.FromOutgoingContext(ctx)
+	req.Header.Add(revactx.TokenHeader, md.Get(revactx.TokenHeader)[0])
+	resp, err := r.dataGatewayClient.Do(req)
+	if err != nil {
+		return []byte{}, err
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return []byte{}, &notFoundErr{}
+	}
+
+	b, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return []byte{}, err
+	}
+
+	if err = resp.Body.Close(); err != nil {
+		return []byte{}, err
+	}
+
+	return b, nil
+}

ocs/pkg/service/v0/groups.go

@@ -22,8 +22,11 @@ import (
 // ListUserGroups lists a users groups
 func (o Ocs) ListUserGroups(w http.ResponseWriter, r *http.Request) {
 	userid := chi.URLParam(r, "userid")
+	userid, err := url.PathUnescape(userid)
+	if err != nil {
+		o.mustRender(w, r, response.ErrRender(data.MetaServerError.StatusCode, err.Error()))
+	}
 	var account *accounts.Account
-	var err error
 
 	// short circuit if there is a user already in the context
 	if u, ok := revactx.ContextGetUser(r.Context()); ok {
@@ -102,14 +105,12 @@ func (o Ocs) ListUserGroups(w http.ResponseWriter, r *http.Request) {
 
 // AddToGroup adds a user to a group
 func (o Ocs) AddToGroup(w http.ResponseWriter, r *http.Request) {
-	err := r.ParseForm()
-	if err != nil {
-		o.mustRender(w, r, response.ErrRender(data.MetaBadRequest.StatusCode, "Could not parse form from request"))
-		return
-	}
-
+	groupid := r.PostFormValue("groupid")
 	userid := chi.URLParam(r, "userid")
-	groupid := r.PostForm.Get("groupid")
+	userid, err := url.PathUnescape(userid)
+	if err != nil {
+		o.mustRender(w, r, response.ErrRender(data.MetaServerError.StatusCode, err.Error()))
+	}
 
 	if groupid == "" {
 		o.mustRender(w, r, response.ErrRender(data.MetaBadRequest.StatusCode, "empty group assignment: unspecified group"))
@@ -161,8 +162,10 @@ func (o Ocs) AddToGroup(w http.ResponseWriter, r *http.Request) {
 // RemoveFromGroup removes a user from a group
 func (o Ocs) RemoveFromGroup(w http.ResponseWriter, r *http.Request) {
 	userid := chi.URLParam(r, "userid")
-
-	var err error
+	userid, err := url.PathUnescape(userid)
+	if err != nil {
+		o.mustRender(w, r, response.ErrRender(data.MetaServerError.StatusCode, err.Error()))
+	}
 
 	// Really? a DELETE with form encoded body?!?
 	// but it is not encoded as mime, so we cannot just call r.ParseForm()
@@ -346,6 +349,10 @@ func (o Ocs) AddGroup(w http.ResponseWriter, r *http.Request) {
 // DeleteGroup deletes a group
 func (o Ocs) DeleteGroup(w http.ResponseWriter, r *http.Request) {
 	groupid := chi.URLParam(r, "groupid")
+	groupid, err := url.PathUnescape(groupid)
+	if err != nil {
+		o.mustRender(w, r, response.ErrRender(data.MetaServerError.StatusCode, err.Error()))
+	}
 
 	// ocs only knows about names so we have to look up the internal id
 	group, err := o.fetchGroupByName(r.Context(), groupid)
@@ -382,6 +389,10 @@ func (o Ocs) DeleteGroup(w http.ResponseWriter, r *http.Request) {
 func (o Ocs) GetGroupMembers(w http.ResponseWriter, r *http.Request) {
 
 	groupid := chi.URLParam(r, "groupid")
+	groupid, err := url.PathUnescape(groupid)
+	if err != nil {
+		o.mustRender(w, r, response.ErrRender(data.MetaServerError.StatusCode, err.Error()))
+	}
 
 	// ocs only knows about names so we have to look up the internal id
 	group, err := o.fetchGroupByName(r.Context(), groupid)

ocs/pkg/service/v0/users.go

@@ -6,6 +6,7 @@ import (
 	"encoding/hex"
 	"fmt"
 	"net/http"
+	"net/url"
 	"strconv"
 	"strings"
 
@@ -85,8 +86,11 @@ func (o Ocs) GetSelf(w http.ResponseWriter, r *http.Request) {
 // GetUser returns the user with the given userid
 func (o Ocs) GetUser(w http.ResponseWriter, r *http.Request) {
 	userid := chi.URLParam(r, "userid")
+	userid, err := url.PathUnescape(userid)
+	if err != nil {
+		o.mustRender(w, r, response.ErrRender(data.MetaServerError.StatusCode, err.Error()))
+	}
 	var account *accounts.Account
-	var err error
 
 	switch {
 	case userid == "":
@@ -273,9 +277,12 @@ func (o Ocs) AddUser(w http.ResponseWriter, r *http.Request) {
 // EditUser creates a new user account
 func (o Ocs) EditUser(w http.ResponseWriter, r *http.Request) {
 	userid := chi.URLParam(r, "userid")
+	userid, err := url.PathUnescape(userid)
+	if err != nil {
+		o.mustRender(w, r, response.ErrRender(data.MetaServerError.StatusCode, err.Error()))
+	}
 
 	var account *accounts.Account
-	var err error
 	switch o.config.AccountBackend {
 	case "accounts":
 		account, err = o.fetchAccountByUsername(r.Context(), userid)
@@ -351,9 +358,12 @@ func (o Ocs) EditUser(w http.ResponseWriter, r *http.Request) {
 // DeleteUser deletes a user
 func (o Ocs) DeleteUser(w http.ResponseWriter, r *http.Request) {
 	userid := chi.URLParam(r, "userid")
+	userid, err := url.PathUnescape(userid)
+	if err != nil {
+		o.mustRender(w, r, response.ErrRender(data.MetaServerError.StatusCode, err.Error()))
+	}
 
 	var account *accounts.Account
-	var err error
 	switch o.config.AccountBackend {
 	case "accounts":
 		account, err = o.fetchAccountByUsername(r.Context(), userid)
@@ -508,9 +518,12 @@ func (o Ocs) mintTokenForUser(ctx context.Context, account *accounts.Account) (s
 // EnableUser enables a user
 func (o Ocs) EnableUser(w http.ResponseWriter, r *http.Request) {
 	userid := chi.URLParam(r, "userid")
+	userid, err := url.PathUnescape(userid)
+	if err != nil {
+		o.mustRender(w, r, response.ErrRender(data.MetaServerError.StatusCode, err.Error()))
+	}
 
 	var account *accounts.Account
-	var err error
 	switch o.config.AccountBackend {
 	case "accounts":
 		account, err = o.fetchAccountByUsername(r.Context(), userid)
@@ -559,9 +572,12 @@ func (o Ocs) EnableUser(w http.ResponseWriter, r *http.Request) {
 // DisableUser disables a user
 func (o Ocs) DisableUser(w http.ResponseWriter, r *http.Request) {
 	userid := chi.URLParam(r, "userid")
+	userid, err := url.PathUnescape(userid)
+	if err != nil {
+		o.mustRender(w, r, response.ErrRender(data.MetaServerError.StatusCode, err.Error()))
+	}
 
 	var account *accounts.Account
-	var err error
 	switch o.config.AccountBackend {
 	case "accounts":
 		account, err = o.fetchAccountByUsername(r.Context(), userid)

storage/pkg/config/config.go

@@ -922,7 +922,7 @@ func DefaultConfig() *Config {
 				},
 				Driver:           "ocis",
 				ExposeDataServer: false,
-				DataServerURL:    "http://localhost:9216",
+				DataServerURL:    "http://localhost:9216/data",
 				TempFolder:       path.Join(defaults.BaseDataPath(), "tmp", "metadata"),
 				DataProvider:     DataProvider{},
 			},

tests/acceptance/expected-failures-webUI-on-OCIS-storage-ocisSmokeTest.md

@@ -10,6 +10,9 @@ Other free text and Markdown formatting can be used elsewhere in the document if
 
 Only the web scenarios tagged ocisSmokeTest are run by default in OCIS CI. This file lists the expected-failures of those ocisSmokeTest scenarios.
 
+### [unexpected behavior when renaming files](https://github.com/owncloud/web/issues/4893)
+- [webUIFilesCopy/copy.feature:36](https://github.com/owncloud/web/blob/master/tests/acceptance/features/webUIFilesCopy/copy.feature#L36)
+
 ### [enable re-sharing is not possible](https://github.com/owncloud/ocis/issues/1743)
 -   [webUISharingFilePermissionMultipleUsers/shareFileWithMultipleUsers.feature:67](https://github.com/owncloud/web/blob/master/tests/acceptance/features/webUISharingFilePermissionMultipleUsers/shareFileWithMultipleUsers.feature#L67)
 -   [webUISharingFilePermissionMultipleUsers/shareFileWithMultipleUsers.feature:66](https://github.com/owncloud/web/blob/master/tests/acceptance/features/webUISharingFilePermissionMultipleUsers/shareFileWithMultipleUsers.feature#L66)

tests/acceptance/expected-failures-webUI-on-OCIS-storage.md

@@ -8,6 +8,13 @@ Level-3 headings should be used for the references to the relevant issues. Inclu
 
 Other free text and markdown formatting can be used elsewhere in the document if needed. But if you want to explain something about the issue, then please post that in the issue itself.
 
+### [unexpected behavior when renaming files](https://github.com/owncloud/web/issues/4893)
+- [webUIFilesCopy/copy.feature:36](https://github.com/owncloud/web/blob/master/tests/acceptance/features/webUIFilesCopy/copy.feature#L36)
+- [webUIFilesCopy/copy.feature:68](https://github.com/owncloud/web/blob/master/tests/acceptance/features/webUIFilesCopy/copy.feature#L68)
+- [webUIFilesCopy/copy.feature:69](https://github.com/owncloud/web/blob/master/tests/acceptance/features/webUIFilesCopy/copy.feature#L69)
+- [webUIFilesCopy/copy.feature:70](https://github.com/owncloud/web/blob/master/tests/acceptance/features/webUIFilesCopy/copy.feature#L70)
+- [webUIFilesCopy/copy.feature:71](https://github.com/owncloud/web/blob/master/tests/acceptance/features/webUIFilesCopy/copy.feature#L71)
+
 ### [Media Viewer does not support mp3 files](https://github.com/owncloud/ocis/issues/1106)
 -   [webUIPreview/imageMediaViewer.feature:84](https://github.com/owncloud/web/blob/master/tests/acceptance/features/webUIPreview/imageMediaViewer.feature#L84)
 -   [webUIPreview/imageMediaViewer.feature:91](https://github.com/owncloud/web/blob/master/tests/acceptance/features/webUIPreview/imageMediaViewer.feature#L91)