[full-ci] Ocm tus (#7998)

* Add service account configuration for the ocm service

* Bump reva

* Add changelog
This commit is contained in:
Andre Duffeck
2023-12-18 10:05:31 +01:00
committed by GitHub
parent 96c0e71b93
commit 5749348b11
16 changed files with 530 additions and 80 deletions
+1
View File
@@ -2,6 +2,7 @@ Enhancement: Add ocm and sciencemesh services
We added sciencemesh and ocm services to enable federation.
https://github.com/owncloud/ocis/pull/7998
https://github.com/owncloud/ocis/pull/7576
https://github.com/owncloud/ocis/pull/7464
https://github.com/owncloud/ocis/pull/7463
+2 -2
View File
@@ -13,7 +13,7 @@ require (
github.com/coreos/go-oidc v2.2.1+incompatible
github.com/coreos/go-oidc/v3 v3.9.0
github.com/cs3org/go-cs3apis v0.0.0-20231023073225-7748710e0781
github.com/cs3org/reva/v2 v2.17.1-0.20231215134723-5142bf31838d
github.com/cs3org/reva/v2 v2.17.1-0.20231215155002-7810e3d8be38
github.com/dhowden/tag v0.0.0-20230630033851-978a0926ee25
github.com/disintegration/imaging v1.6.2
github.com/dutchcoders/go-clamd v0.0.0-20170520113014-b970184f4d9e
@@ -349,4 +349,4 @@ require (
replace github.com/go-micro/plugins/v4/store/nats-js-kv => github.com/kobergj/plugins/v4/store/nats-js-kv v0.0.0-20231207143248-4d424e3ae348
replace github.com/studio-b12/gowebdav => github.com/aduffeck/gowebdav v0.0.0-20231123085457-ff658b6ea159
replace github.com/studio-b12/gowebdav => github.com/aduffeck/gowebdav v0.0.0-20231215074047-b00689b28e5f
+4 -6
View File
@@ -827,8 +827,8 @@ github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWX
github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
github.com/acomagu/bufpipe v1.0.3 h1:fxAGrHZTgQ9w5QqVItgzwj235/uYZYgbXitB+dLupOk=
github.com/acomagu/bufpipe v1.0.3/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4=
github.com/aduffeck/gowebdav v0.0.0-20231123085457-ff658b6ea159 h1:m63hhLqbqmLGGPtyTtjTdxae61d9tMbRdKvMaDHWcDs=
github.com/aduffeck/gowebdav v0.0.0-20231123085457-ff658b6ea159/go.mod h1:bHA7t77X/QFExdeAnDzK6vKM34kEZAcE1OX4MfiwjkE=
github.com/aduffeck/gowebdav v0.0.0-20231215074047-b00689b28e5f h1:rxzQfsnLmEm5YnAf0KDoTmswnnTX9whwAsFT7n1I1kk=
github.com/aduffeck/gowebdav v0.0.0-20231215074047-b00689b28e5f/go.mod h1:bHA7t77X/QFExdeAnDzK6vKM34kEZAcE1OX4MfiwjkE=
github.com/agnivade/levenshtein v1.1.1 h1:QY8M92nrzkmr798gCo3kmMyqXFzdQVpxLlGPRBij0P8=
github.com/agnivade/levenshtein v1.1.1/go.mod h1:veldBMzWxcCG2ZvUTKD2kJNRdCk5hVbJomOvKkmgYbo=
github.com/ajg/form v1.5.1 h1:t9c7v8JUKu/XxOGBU0yjNpaMloxGEJhUkqFRq0ibGeU=
@@ -1021,10 +1021,8 @@ github.com/crewjam/saml v0.4.14 h1:g9FBNx62osKusnFzs3QTN5L9CVA/Egfgm+stJShzw/c=
github.com/crewjam/saml v0.4.14/go.mod h1:UVSZCf18jJkk6GpWNVqcyQJMD5HsRugBPf4I1nl2mME=
github.com/cs3org/go-cs3apis v0.0.0-20231023073225-7748710e0781 h1:BUdwkIlf8IS2FasrrPg8gGPHQPOrQ18MS1Oew2tmGtY=
github.com/cs3org/go-cs3apis v0.0.0-20231023073225-7748710e0781/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY=
github.com/cs3org/reva/v2 v2.17.1-0.20231215113433-48c0ea55bf47 h1:6DfMeFpCXoqlfm/+FJ/mFs8Ul5WCZNlorsbDM9Z/ATE=
github.com/cs3org/reva/v2 v2.17.1-0.20231215113433-48c0ea55bf47/go.mod h1:oX1YtLKGr7jatGk0CpPM4GKbSEIdHhmsQuSAYElnN1U=
github.com/cs3org/reva/v2 v2.17.1-0.20231215134723-5142bf31838d h1:OYkjbcOAntD5JBMAuyj+bR1bg5jM+BjRvFBiTlmnxWQ=
github.com/cs3org/reva/v2 v2.17.1-0.20231215134723-5142bf31838d/go.mod h1:JyvlRw2v8BTH5t+ISj1Yc+EMDBcyf8LMB5o98HufWis=
github.com/cs3org/reva/v2 v2.17.1-0.20231215155002-7810e3d8be38 h1:vkA/Ty82yETTDrpLV/b5/9VXUJ/9o7vTRs7ampZC5LU=
github.com/cs3org/reva/v2 v2.17.1-0.20231215155002-7810e3d8be38/go.mod h1:5Yxh1DneWZQvMBOiBVv3LJaSsSlRCoqCBa4Wws7PWHw=
github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+12 -5
View File
@@ -17,11 +17,12 @@ type Config struct {
Log *Log `yaml:"log"`
Debug Debug `yaml:"debug"`
HTTP HTTPConfig `yaml:"http"`
Middleware Middleware `yaml:"middleware"`
GRPC GRPCConfig `yaml:"grpc"`
GRPCClientTLS *shared.GRPCClientTLS `yaml:"grpc_client_tls"`
GrpcClient client.Client `yaml:"-"`
HTTP HTTPConfig `yaml:"http"`
Middleware Middleware `yaml:"middleware"`
GRPC GRPCConfig `yaml:"grpc"`
GRPCClientTLS *shared.GRPCClientTLS `yaml:"grpc_client_tls"`
GrpcClient client.Client `yaml:"-"`
ServiceAccount ServiceAccount `yaml:"service_account"`
Reva *shared.Reva `yaml:"reva"`
OCMD OCMD `yaml:"ocmd"`
@@ -56,6 +57,12 @@ type Auth struct {
CredentialsByUserAgent map[string]string `yaml:"credentials_by_user_agent"`
}
// ServiceAccount is the configuration for the used service account
type ServiceAccount struct {
ID string `yaml:"service_account_id" env:"OCIS_SERVICE_ACCOUNT_ID;OCM_SERVICE_ACCOUNT_ID" desc:"The ID of the service account the service should use. See the 'auth-service' service description for more details."`
Secret string `yaml:"service_account_secret" env:"OCIS_SERVICE_ACCOUNT_SECRET;OCM_SERVICE_ACCOUNT_SECRET" desc:"The service account secret."`
}
// CORS defines the available cors configuration.
type CORS struct {
AllowedOrigins []string `yaml:"allow_origins" env:"OCIS_CORS_ALLOW_ORIGINS;OCM_CORS_ALLOW_ORIGINS" desc:"A list of allowed CORS origins. See following chapter for more details: *Access-Control-Allow-Origin* at https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Access-Control-Allow-Origin. See the Environment Variable Types description for more details."`
+4 -1
View File
@@ -56,7 +56,10 @@ func OCMConfigFromStruct(cfg *config.Config, logger log.Logger) map[string]inter
"driver": "ocmreceived",
"drivers": map[string]interface{}{
"ocmreceived": map[string]interface{}{
"insecure": cfg.OCMStorageProvider.Insecure,
"insecure": cfg.OCMStorageProvider.Insecure,
"storage_root": cfg.OCMStorageProvider.StorageRoot,
"service_account_id": cfg.ServiceAccount.ID,
"service_account_secret": cfg.ServiceAccount.Secret,
},
},
"data_txs": map[string]interface{}{
@@ -515,6 +515,14 @@ func (s *service) UpdateReceivedOCMShare(ctx context.Context, req *ocm.UpdateRec
func (s *service) GetReceivedOCMShare(ctx context.Context, req *ocm.GetReceivedOCMShareRequest) (*ocm.GetReceivedOCMShareResponse, error) {
user := ctxpkg.ContextMustGetUser(ctx)
if user.Id.GetType() == userpb.UserType_USER_TYPE_SERVICE {
var uid userpb.UserId
_ = utils.ReadJSONFromOpaque(req.Opaque, "userid", &uid)
user = &userpb.User{
Id: &uid,
}
}
ocmshare, err := s.repo.GetReceivedShare(ctx, user, req.Ref)
if err != nil {
if errors.Is(err, share.ErrShareNotFound) {
@@ -523,7 +531,7 @@ func (s *service) GetReceivedOCMShare(ctx context.Context, req *ocm.GetReceivedO
}, nil
}
return &ocm.GetReceivedOCMShareResponse{
Status: status.NewInternal(ctx, "error getting received share"),
Status: status.NewInternal(ctx, "error getting received share: "+err.Error()),
}, nil
}
@@ -68,6 +68,7 @@ type config struct {
}
type passwordPolicy struct {
Disabled bool `mapstructure:"disabled"`
MinCharacters int `mapstructure:"min_characters"`
MinLowerCaseCharacters int `mapstructure:"min_lowercase_characters"`
MinUpperCaseCharacters int `mapstructure:"min_uppercase_characters"`
@@ -173,9 +174,10 @@ func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) {
func newPasswordPolicy(c *passwordPolicy) password.Validator {
if c == nil {
return password.NewPasswordPolicy(0, 0, 0, 0, 0, nil)
return password.NewPasswordPolicy(true, 0, 0, 0, 0, 0, nil)
}
return password.NewPasswordPolicy(
c.Disabled,
c.MinCharacters,
c.MinLowerCaseCharacters,
c.MinUpperCaseCharacters,
@@ -137,6 +137,7 @@ func (s *svc) handlePut(ctx context.Context, w http.ResponseWriter, r *http.Requ
length, err := getContentLength(w, r)
if err != nil {
log.Error().Err(err).Msg("error getting the content length")
w.WriteHeader(http.StatusBadRequest)
return
}
@@ -117,6 +117,7 @@ type CapabilitiesGraph struct {
// CapabilitiesPasswordPolicy hold the password policy capabilities
type CapabilitiesPasswordPolicy struct {
Disabled bool `json:"disabled" xml:"disabled" mapstructure:"disabled"`
MinCharacters int `json:"min_characters" xml:"min_characters" mapstructure:"min_characters"`
MaxCharacters int `json:"max_characters" xml:"max_characters" mapstructure:"max_characters"`
MinLowerCaseCharacters int `json:"min_lowercase_characters" xml:"min_lowercase_characters" mapstructure:"min_lowercase_characters"`
@@ -1734,9 +1734,10 @@ func publicPwdEnforced(c *config.Config) passwordEnforced {
func passwordPolicies(c *config.Config) password.Validator {
if c.Capabilities.Capabilities == nil || c.Capabilities.Capabilities.PasswordPolicy == nil {
return password.NewPasswordPolicy(0, 0, 0, 0, 0, nil)
return password.NewPasswordPolicy(true, 0, 0, 0, 0, 0, nil)
}
return password.NewPasswordPolicy(
c.Capabilities.Capabilities.PasswordPolicy.Disabled,
c.Capabilities.Capabilities.PasswordPolicy.MinCharacters,
c.Capabilities.Capabilities.PasswordPolicy.MinLowerCaseCharacters,
c.Capabilities.Capabilities.PasswordPolicy.MinUpperCaseCharacters,
+30 -43
View File
@@ -30,6 +30,7 @@ import (
"strings"
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
userv1beta1 "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1"
ocmpb "github.com/cs3org/go-cs3apis/cs3/sharing/ocm/v1beta1"
@@ -59,8 +60,11 @@ type driver struct {
}
type config struct {
GatewaySVC string `mapstructure:"gatewaysvc"`
Insecure bool `mapstructure:"insecure"`
GatewaySVC string `mapstructure:"gatewaysvc"`
Insecure bool `mapstructure:"insecure"`
StorageRoot string `mapstructure:"storage_root"`
ServiceAccountID string `mapstructure:"service_account_id"`
ServiceAccountSecret string `mapstructure:"service_account_secret"`
}
func (c *config) ApplyDefaults() {
@@ -136,15 +140,19 @@ func shareInfoFromReference(ref *provider.Reference) (*ocmpb.ShareId, string) {
}
func (d *driver) getWebDAVFromShare(ctx context.Context, shareID *ocmpb.ShareId) (*ocmpb.ReceivedShare, string, string, error) {
func (d *driver) getWebDAVFromShare(ctx context.Context, forUser *userpb.UserId, shareID *ocmpb.ShareId) (*ocmpb.ReceivedShare, string, string, error) {
// TODO: we may want to cache the share
res, err := d.gateway.GetReceivedOCMShare(ctx, &ocmpb.GetReceivedOCMShareRequest{
req := &ocmpb.GetReceivedOCMShareRequest{
Ref: &ocmpb.ShareReference{
Spec: &ocmpb.ShareReference_Id{
Id: shareID,
},
},
})
}
if forUser != nil {
req.Opaque = utils.AppendJSONToOpaque(nil, "userid", forUser)
}
res, err := d.gateway.GetReceivedOCMShare(ctx, req)
if err != nil {
return nil, "", "", err
}
@@ -173,10 +181,10 @@ func getWebDAVProtocol(protocols []*ocmpb.Protocol) (*ocmpb.WebDAVProtocol, bool
return nil, false
}
func (d *driver) webdavClient(ctx context.Context, ref *provider.Reference) (*gowebdav.Client, *ocmpb.ReceivedShare, string, error) {
func (d *driver) webdavClient(ctx context.Context, forUser *userpb.UserId, ref *provider.Reference) (*gowebdav.Client, *ocmpb.ReceivedShare, string, error) {
id, rel := shareInfoFromReference(ref)
share, endpoint, secret, err := d.getWebDAVFromShare(ctx, id)
share, endpoint, secret, err := d.getWebDAVFromShare(ctx, forUser, id)
if err != nil {
return nil, nil, "", err
}
@@ -199,7 +207,7 @@ func (d *driver) webdavClient(ctx context.Context, ref *provider.Reference) (*go
}
func (d *driver) CreateDir(ctx context.Context, ref *provider.Reference) error {
client, _, rel, err := d.webdavClient(ctx, ref)
client, _, rel, err := d.webdavClient(ctx, nil, ref)
if err != nil {
return err
}
@@ -207,7 +215,7 @@ func (d *driver) CreateDir(ctx context.Context, ref *provider.Reference) error {
}
func (d *driver) Delete(ctx context.Context, ref *provider.Reference) error {
client, _, rel, err := d.webdavClient(ctx, ref)
client, _, rel, err := d.webdavClient(ctx, nil, ref)
if err != nil {
return err
}
@@ -215,7 +223,7 @@ func (d *driver) Delete(ctx context.Context, ref *provider.Reference) error {
}
func (d *driver) TouchFile(ctx context.Context, ref *provider.Reference, markprocessing bool, mtime string) error {
client, _, rel, err := d.webdavClient(ctx, ref)
client, _, rel, err := d.webdavClient(ctx, nil, ref)
if err != nil {
return err
}
@@ -223,7 +231,7 @@ func (d *driver) TouchFile(ctx context.Context, ref *provider.Reference, markpro
}
func (d *driver) Move(ctx context.Context, oldRef, newRef *provider.Reference) error {
client, _, relOld, err := d.webdavClient(ctx, oldRef)
client, _, relOld, err := d.webdavClient(ctx, nil, oldRef)
if err != nil {
return err
}
@@ -263,7 +271,7 @@ func convertStatToResourceInfo(ref *provider.Reference, f fs.FileInfo, share *oc
}
webdavProtocol, _ := getWebDAVProtocol(share.Protocols)
return &provider.ResourceInfo{
ri := provider.ResourceInfo{
Type: t,
Id: id,
MimeType: mime.Detect(f.IsDir(), f.Name()),
@@ -278,11 +286,17 @@ func convertStatToResourceInfo(ref *provider.Reference, f fs.FileInfo, share *oc
Checksum: &provider.ResourceChecksum{
Type: provider.ResourceChecksumType_RESOURCE_CHECKSUM_TYPE_INVALID,
},
}, nil
}
if f.(gowebdav.File).StatusCode() == 425 {
ri.Opaque = utils.AppendPlainToOpaque(ri.Opaque, "status", "processing")
}
return &ri, nil
}
func (d *driver) GetMD(ctx context.Context, ref *provider.Reference, _ []string, _ []string) (*provider.ResourceInfo, error) {
client, share, rel, err := d.webdavClient(ctx, ref)
client, share, rel, err := d.webdavClient(ctx, nil, ref)
if err != nil {
return nil, err
}
@@ -299,7 +313,7 @@ func (d *driver) GetMD(ctx context.Context, ref *provider.Reference, _ []string,
}
func (d *driver) ListFolder(ctx context.Context, ref *provider.Reference, _ []string, _ []string) ([]*provider.ResourceInfo, error) {
client, share, rel, err := d.webdavClient(ctx, ref)
client, share, rel, err := d.webdavClient(ctx, nil, ref)
if err != nil {
return nil, err
}
@@ -320,35 +334,8 @@ func (d *driver) ListFolder(ctx context.Context, ref *provider.Reference, _ []st
return res, nil
}
func (d *driver) InitiateUpload(ctx context.Context, ref *provider.Reference, _ int64, _ map[string]string) (map[string]string, error) {
shareID, rel := shareInfoFromReference(ref)
p := getPathFromShareIDAndRelPath(shareID, rel)
return map[string]string{
"simple": p,
}, nil
}
func (d *driver) Upload(ctx context.Context, req storage.UploadRequest, _ storage.UploadFinishedFunc) (provider.ResourceInfo, error) {
client, _, rel, err := d.webdavClient(ctx, req.Ref)
if err != nil {
return provider.ResourceInfo{}, err
}
client.SetInterceptor(func(method string, rq *http.Request) {
// Set the content length on the request struct directly instead of the header.
// The content-length header gets reset by the golang http library before
// sendind out the request, resulting in chunked encoding to be used which
// breaks the quota checks in ocdav.
if method == "PUT" {
rq.ContentLength = req.Length
}
})
return provider.ResourceInfo{}, client.WriteStream(rel, req.Body, 0)
}
func (d *driver) Download(ctx context.Context, ref *provider.Reference) (io.ReadCloser, error) {
client, _, rel, err := d.webdavClient(ctx, ref)
client, _, rel, err := d.webdavClient(ctx, nil, ref)
if err != nil {
return nil, err
}
+405
View File
@@ -0,0 +1,405 @@
// Copyright 2018-2023 CERN
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// In applying this license, CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.
package ocm
import (
"context"
"crypto/md5"
"crypto/sha1"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"hash"
"hash/adler32"
"io"
"net/http"
"os"
"path/filepath"
"strings"
"github.com/google/uuid"
tusd "github.com/tus/tusd/pkg/handler"
userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
"github.com/cs3org/reva/v2/pkg/appctx"
ctxpkg "github.com/cs3org/reva/v2/pkg/ctx"
"github.com/cs3org/reva/v2/pkg/errtypes"
"github.com/cs3org/reva/v2/pkg/storage"
"github.com/cs3org/reva/v2/pkg/utils"
)
var defaultFilePerm = os.FileMode(0664)
func (d *driver) ListUploadSessions(ctx context.Context, filter storage.UploadSessionFilter) ([]storage.UploadSession, error) {
return []storage.UploadSession{}, nil
}
func (d *driver) InitiateUpload(ctx context.Context, ref *provider.Reference, uploadLength int64, metadata map[string]string) (map[string]string, error) {
shareID, rel := shareInfoFromReference(ref)
p := getPathFromShareIDAndRelPath(shareID, rel)
info := tusd.FileInfo{
MetaData: tusd.MetaData{
"filename": filepath.Base(p),
"dir": filepath.Dir(p),
},
Size: uploadLength,
}
upload, err := d.NewUpload(ctx, info)
if err != nil {
return nil, err
}
info, _ = upload.GetInfo(ctx)
return map[string]string{
"simple": info.ID,
"tus": info.ID,
}, nil
}
func (d *driver) Upload(ctx context.Context, req storage.UploadRequest, _ storage.UploadFinishedFunc) (provider.ResourceInfo, error) {
shareID, _ := shareInfoFromReference(req.Ref)
u, err := d.GetUpload(ctx, shareID.OpaqueId)
if err != nil {
return provider.ResourceInfo{}, err
}
info, err := u.GetInfo(ctx)
if err != nil {
return provider.ResourceInfo{}, err
}
client, _, rel, err := d.webdavClient(ctx, nil, &provider.Reference{
Path: filepath.Join(info.MetaData["dir"], info.MetaData["filename"]),
})
if err != nil {
return provider.ResourceInfo{}, err
}
client.SetInterceptor(func(method string, rq *http.Request) {
// Set the content length on the request struct directly instead of the header.
// The content-length header gets reset by the golang http library before
// sendind out the request, resulting in chunked encoding to be used which
// breaks the quota checks in ocdav.
if method == "PUT" {
rq.ContentLength = req.Length
}
})
return provider.ResourceInfo{}, client.WriteStream(rel, req.Body, 0)
}
// UseIn tells the tus upload middleware which extensions it supports.
func (d *driver) UseIn(composer *tusd.StoreComposer) {
composer.UseCore(d)
composer.UseTerminater(d)
composer.UseConcater(d)
composer.UseLengthDeferrer(d)
}
// AsTerminatableUpload returns a TerminatableUpload
// To implement the termination extension as specified in https://tus.io/protocols/resumable-upload.html#termination
// the storage needs to implement AsTerminatableUpload
func (d *driver) AsTerminatableUpload(up tusd.Upload) tusd.TerminatableUpload {
return up.(*upload)
}
// AsLengthDeclarableUpload returns a LengthDeclarableUpload
// To implement the creation-defer-length extension as specified in https://tus.io/protocols/resumable-upload.html#creation
// the storage needs to implement AsLengthDeclarableUpload
func (d *driver) AsLengthDeclarableUpload(up tusd.Upload) tusd.LengthDeclarableUpload {
return up.(*upload)
}
// AsConcatableUpload returns a ConcatableUpload
// To implement the concatenation extension as specified in https://tus.io/protocols/resumable-upload.html#concatenation
// the storage needs to implement AsConcatableUpload
func (d *driver) AsConcatableUpload(up tusd.Upload) tusd.ConcatableUpload {
return up.(*upload)
}
// To implement the core tus.io protocol as specified in https://tus.io/protocols/resumable-upload.html#core-protocol
// - the storage needs to implement NewUpload and GetUpload
// - the upload needs to implement the tusd.Upload interface: WriteChunk, GetInfo, GetReader and FinishUpload
// NewUpload returns a new tus Upload instance
func (d *driver) NewUpload(ctx context.Context, info tusd.FileInfo) (tusd.Upload, error) {
return NewUpload(ctx, d, d.c.StorageRoot, info)
}
// GetUpload returns the Upload for the given upload id
func (d *driver) GetUpload(ctx context.Context, id string) (tusd.Upload, error) {
return GetUpload(ctx, d, d.c.StorageRoot, id)
}
func NewUpload(ctx context.Context, d *driver, storageRoot string, info tusd.FileInfo) (tusd.Upload, error) {
if info.MetaData["filename"] == "" {
return nil, errors.New("Decomposedfs: missing filename in metadata")
}
if info.MetaData["dir"] == "" {
return nil, errors.New("Decomposedfs: missing dir in metadata")
}
uploadRoot := filepath.Join(storageRoot, "uploads")
info.ID = uuid.New().String()
user, ok := ctxpkg.ContextGetUser(ctx)
if !ok {
return nil, errors.New("no user in context")
}
info.MetaData["user"] = user.GetId().GetOpaqueId()
info.MetaData["idp"] = user.GetId().GetIdp()
info.Storage = map[string]string{
"Type": "OCM",
"Path": uploadRoot,
}
u := &upload{
Info: info,
Ctx: ctx,
d: d,
}
err := os.MkdirAll(uploadRoot, 0755)
if err != nil {
return nil, err
}
file, err := os.OpenFile(u.BinPath(), os.O_CREATE|os.O_WRONLY, defaultFilePerm)
if err != nil {
return nil, err
}
defer file.Close()
err = u.Persist()
if err != nil {
return nil, err
}
return u, nil
}
func GetUpload(ctx context.Context, d *driver, storageRoot string, id string) (tusd.Upload, error) {
info := tusd.FileInfo{}
data, err := os.ReadFile(filepath.Join(storageRoot, "uploads", id+".info"))
if err != nil {
return nil, err
}
err = json.Unmarshal(data, &info)
if err != nil {
return nil, err
}
upload := &upload{
Info: info,
Ctx: ctx,
d: d,
}
return upload, nil
}
type upload struct {
Info tusd.FileInfo
Ctx context.Context
d *driver
}
func (u *upload) InfoPath() string {
return filepath.Join(u.Info.Storage["Path"], u.Info.ID+".info")
}
func (u *upload) BinPath() string {
return filepath.Join(u.Info.Storage["Path"], u.Info.ID)
}
func (u *upload) Persist() error {
data, err := json.Marshal(u.Info)
if err != nil {
return err
}
return os.WriteFile(u.InfoPath(), data, defaultFilePerm)
}
func (u *upload) WriteChunk(ctx context.Context, offset int64, src io.Reader) (int64, error) {
file, err := os.OpenFile(u.BinPath(), os.O_WRONLY|os.O_APPEND, defaultFilePerm)
if err != nil {
return 0, err
}
defer file.Close()
// calculate cheksum here? needed for the TUS checksum extension. https://tus.io/protocols/resumable-upload.html#checksum
// TODO but how do we get the `Upload-Checksum`? WriteChunk() only has a context, offset and the reader ...
// It is sent with the PATCH request, well or in the POST when the creation-with-upload extension is used
// but the tus handler uses a context.Background() so we cannot really check the header and put it in the context ...
n, err := io.Copy(file, src)
// If the HTTP PATCH request gets interrupted in the middle (e.g. because
// the user wants to pause the upload), Go's net/http returns an io.ErrUnexpectedEOF.
// However, for the ocis driver it's not important whether the stream has ended
// on purpose or accidentally.
if err != nil && err != io.ErrUnexpectedEOF {
return n, err
}
u.Info.Offset += n
return n, u.Persist()
}
func (u *upload) GetInfo(ctx context.Context) (tusd.FileInfo, error) {
return u.Info, nil
}
func (u *upload) GetReader(ctx context.Context) (io.Reader, error) {
return os.Open(u.BinPath())
}
func (u *upload) FinishUpload(ctx context.Context) error {
log := appctx.GetLogger(u.Ctx)
// calculate the checksum of the written bytes
// they will all be written to the metadata later, so we cannot omit any of them
// TODO only calculate the checksum in sync that was requested to match, the rest could be async ... but the tests currently expect all to be present
// TODO the hashes all implement BinaryMarshaler so we could try to persist the state for resumable upload. we would neet do keep track of the copied bytes ...
sha1h := sha1.New()
md5h := md5.New()
adler32h := adler32.New()
{
f, err := os.Open(u.BinPath())
if err != nil {
// we can continue if no oc checksum header is set
log.Info().Err(err).Str("binPath", u.BinPath()).Msg("error opening binPath")
}
defer f.Close()
r1 := io.TeeReader(f, sha1h)
r2 := io.TeeReader(r1, md5h)
_, err = io.Copy(adler32h, r2)
if err != nil {
log.Info().Err(err).Msg("error copying checksums")
}
}
// compare if they match the sent checksum
// TODO the tus checksum extension would do this on every chunk, but I currently don't see an easy way to pass in the requested checksum. for now we do it in FinishUpload which is also called for chunked uploads
if u.Info.MetaData["checksum"] != "" {
var err error
parts := strings.SplitN(u.Info.MetaData["checksum"], " ", 2)
if len(parts) != 2 {
return errtypes.BadRequest("invalid checksum format. must be '[algorithm] [checksum]'")
}
switch parts[0] {
case "sha1":
err = u.checkHash(parts[1], sha1h)
case "md5":
err = u.checkHash(parts[1], md5h)
case "adler32":
err = u.checkHash(parts[1], adler32h)
default:
err = errtypes.BadRequest("unsupported checksum algorithm: " + parts[0])
}
if err != nil {
u.cleanup()
return err
}
}
// send to the remote storage via webdav
// shareID, rel := shareInfoFromReference(u.Info.MetaData["ref"])
// p := getPathFromShareIDAndRelPath(shareID, rel)
serviceUserCtx, err := utils.GetServiceUserContext(u.d.c.ServiceAccountID, u.d.gateway, u.d.c.ServiceAccountSecret)
if err != nil {
return err
}
client, _, rel, err := u.d.webdavClient(serviceUserCtx, &userpb.UserId{
OpaqueId: u.Info.MetaData["user"],
Idp: u.Info.MetaData["idp"],
}, &provider.Reference{
Path: filepath.Join(u.Info.MetaData["dir"], u.Info.MetaData["filename"]),
})
if err != nil {
u.cleanup()
return err
}
client.SetInterceptor(func(method string, rq *http.Request) {
// Set the content length on the request struct directly instead of the header.
// The content-length header gets reset by the golang http library before
// sendind out the request, resulting in chunked encoding to be used which
// breaks the quota checks in ocdav.
if method == "PUT" {
rq.ContentLength = u.Info.Size
}
})
f, err := os.Open(u.BinPath())
if err != nil {
return err
}
defer f.Close()
return client.WriteStream(rel, f, 0)
}
func (u *upload) cleanup() {
_ = os.Remove(u.BinPath())
_ = os.Remove(u.InfoPath())
}
func (u *upload) Terminate(ctx context.Context) error {
u.cleanup()
return nil
}
func (u *upload) ConcatUploads(_ context.Context, uploads []tusd.Upload) error {
file, err := os.OpenFile(u.BinPath(), os.O_WRONLY|os.O_APPEND, defaultFilePerm)
if err != nil {
return err
}
defer file.Close()
for _, partialUpload := range uploads {
fileUpload := partialUpload.(*upload)
src, err := os.Open(fileUpload.BinPath())
if err != nil {
return err
}
defer src.Close()
if _, err := io.Copy(file, src); err != nil {
return err
}
}
return nil
}
func (u *upload) DeclareLength(ctx context.Context, length int64) error {
u.Info.Size = length
u.Info.SizeIsDeferred = false
return nil
}
func (u *upload) checkHash(expected string, h hash.Hash) error {
if expected != hex.EncodeToString(h.Sum(nil)) {
return errtypes.ChecksumMismatch(fmt.Sprintf("invalid checksum: expected %s got %x", u.Info.MetaData["checksum"], h.Sum(nil)))
}
return nil
}
+6 -1
View File
@@ -18,6 +18,7 @@ type Validator interface {
// Policies represents a password validation rules
type Policies struct {
disabled bool
minCharacters int
minLowerCaseCharacters int
minUpperCaseCharacters int
@@ -29,8 +30,9 @@ type Policies struct {
}
// NewPasswordPolicy returns a new NewPasswordPolicy instance
func NewPasswordPolicy(minCharacters, minLowerCaseCharacters, minUpperCaseCharacters, minDigits, minSpecialCharacters int, bannedPasswordsList map[string]struct{}) Validator {
func NewPasswordPolicy(disabled bool, minCharacters, minLowerCaseCharacters, minUpperCaseCharacters, minDigits, minSpecialCharacters int, bannedPasswordsList map[string]struct{}) Validator {
p := &Policies{
disabled: disabled,
minCharacters: minCharacters,
minLowerCaseCharacters: minLowerCaseCharacters,
minUpperCaseCharacters: minUpperCaseCharacters,
@@ -46,6 +48,9 @@ func NewPasswordPolicy(minCharacters, minLowerCaseCharacters, minUpperCaseCharac
// Validate implements a password validation regarding the policy
func (s Policies) Validate(str string) error {
if s.disabled {
return nil
}
var allErr error
if !utf8.ValidString(str) {
return fmt.Errorf("the password contains invalid characters")
+34 -12
View File
@@ -159,47 +159,63 @@ func (p *propstat) Modified() time.Time {
return time.Unix(0, 0)
}
func (p *propstat) StatusCode() int {
parts := strings.Split(p.Status, " ")
if len(parts) < 2 {
return -1
}
code, err := strconv.Atoi(parts[1])
if err != nil {
return -1
}
return code
}
type response struct {
Href string `xml:"DAV: href"`
Propstats []propstat `xml:"DAV: propstat"`
}
func getPropstat(r *response, status string) *propstat {
func getPropstat(r *response, statuses []string) *propstat {
for _, prop := range r.Propstats {
if strings.Contains(prop.Status, status) {
return &prop
for _, status := range statuses {
if strings.Contains(prop.Status, status) {
return &prop
}
}
}
return nil
}
// ReadDir reads the contents of a remote directory
func (c *Client) ReadDir(path string) ([]os.FileInfo, error) {
func (c *Client) ReadDir(path string) ([]FileInfo, error) {
return c.ReadDirWithProps(path, defaultProps)
}
// ReadDirWithProps reads the contents of the directory at the given path, along with the specified properties.
func (c *Client) ReadDirWithProps(path string, props []string) ([]os.FileInfo, error) {
func (c *Client) ReadDirWithProps(path string, props []string) ([]FileInfo, error) {
propfindprops := ""
if len(props) > 0 {
propfindprops = `<d:prop><d:` + strings.Join(props, "/><d:") + `/></d:prop>`
}
files := make([]os.FileInfo, 0)
files := make([]FileInfo, 0)
skipSelf := true
parse := func(resp interface{}) error {
r := resp.(*response)
if skipSelf {
skipSelf = false
if p := getPropstat(r, "200"); p != nil && p.Type() == "collection" {
if p := getPropstat(r, []string{"200", "425"}); p != nil && p.Type() == "collection" {
r.Propstats = nil
return nil
}
return NewPathError("ReadDir", path, 405)
}
if p := getPropstat(r, "200"); p != nil {
if p := getPropstat(r, []string{"200", "425"}); p != nil {
var name string
if ps, err := url.PathUnescape(r.Href); err == nil {
name = pathpkg.Base(ps)
@@ -227,17 +243,19 @@ func (c *Client) ReadDirWithProps(path string, props []string) ([]os.FileInfo, e
}
// Stat returns the file stats for a specified path with the default properties
func (c *Client) Stat(path string) (os.FileInfo, error) {
func (c *Client) Stat(path string) (FileInfo, error) {
return c.StatWithProps(path, defaultProps)
}
// StatWithProps returns the FileInfo for the specified path along with the specified properties.
func (c *Client) StatWithProps(path string, props []string) (os.FileInfo, error) {
func (c *Client) StatWithProps(path string, props []string) (FileInfo, error) {
var f *File
parse := func(resp interface{}) error {
r := resp.(*response)
if p := getPropstat(r, "200"); p != nil && f == nil {
if p := getPropstat(r, []string{"200", "425"}); p != nil && f == nil {
f = newFile(".", path, p)
} else {
return NewPathError("StatWithProps", path, 404)
}
r.Propstats = nil
@@ -257,10 +275,14 @@ func (c *Client) StatWithProps(path string, props []string) (os.FileInfo, error)
if err != nil {
if _, ok := err.(*os.PathError); !ok {
return nil, NewPathErrorErr("ReadDir", path, err)
return nil, NewPathErrorErr("StatWithProps", path, err)
}
return nil, err
}
if f == nil {
return nil, NewPathError("StatWithProps", path, 404)
}
return *f, err
}
+13 -4
View File
@@ -7,6 +7,11 @@ import (
"time"
)
type FileInfo interface {
os.FileInfo
StatusCode() int
}
// File is our structure for a given file
type File struct {
path string
@@ -16,12 +21,13 @@ type File struct {
modified time.Time
etag string
isdir bool
props Props
propstat propstat
status int
}
func newFile(path, name string, p *propstat) *File {
f := &File{
props: p.Props,
propstat: *p,
}
path = FixSlashes(path)
@@ -30,7 +36,6 @@ func newFile(path, name string, p *propstat) *File {
f.modified = p.Modified()
f.etag = p.ETag()
f.contentType = p.ContentType()
f.props = p.Props
if p.Type() == "collection" {
f.path = filepath.Clean(f.path + "/")
@@ -90,7 +95,11 @@ func (f File) IsDir() bool {
// Sys ????
func (f File) Sys() interface{} {
return f.props
return f.propstat.Props
}
func (f File) StatusCode() int {
return f.propstat.StatusCode()
}
// String lets us see file information
+3 -3
View File
@@ -362,7 +362,7 @@ github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1
github.com/cs3org/go-cs3apis/cs3/storage/registry/v1beta1
github.com/cs3org/go-cs3apis/cs3/tx/v1beta1
github.com/cs3org/go-cs3apis/cs3/types/v1beta1
# github.com/cs3org/reva/v2 v2.17.1-0.20231215134723-5142bf31838d
# github.com/cs3org/reva/v2 v2.17.1-0.20231215155002-7810e3d8be38
## explicit; go 1.21
github.com/cs3org/reva/v2/cmd/revad/internal/grace
github.com/cs3org/reva/v2/cmd/revad/runtime
@@ -1721,7 +1721,7 @@ github.com/stretchr/objx
github.com/stretchr/testify/assert
github.com/stretchr/testify/mock
github.com/stretchr/testify/require
# github.com/studio-b12/gowebdav v0.0.0-20221015232716-17255f2e7423 => github.com/aduffeck/gowebdav v0.0.0-20231123085457-ff658b6ea159
# github.com/studio-b12/gowebdav v0.0.0-20221015232716-17255f2e7423 => github.com/aduffeck/gowebdav v0.0.0-20231215074047-b00689b28e5f
## explicit; go 1.17
github.com/studio-b12/gowebdav
# github.com/tchap/go-patricia/v2 v2.3.1
@@ -2298,4 +2298,4 @@ stash.kopano.io/kgol/oidc-go
## explicit; go 1.13
stash.kopano.io/kgol/rndm
# github.com/go-micro/plugins/v4/store/nats-js-kv => github.com/kobergj/plugins/v4/store/nats-js-kv v0.0.0-20231207143248-4d424e3ae348
# github.com/studio-b12/gowebdav => github.com/aduffeck/gowebdav v0.0.0-20231123085457-ff658b6ea159
# github.com/studio-b12/gowebdav => github.com/aduffeck/gowebdav v0.0.0-20231215074047-b00689b28e5f