From e5c3f93c85872a9ea7c14adcca6120e140786071 Mon Sep 17 00:00:00 2001 From: Roman Perekhod Date: Wed, 3 Jan 2024 12:42:31 +0100 Subject: [PATCH] [full-ci] [bump reva] fix the upload postprocessing --- .../unreleased/fix-upload-postprocessing.md | 6 ++++ go.mod | 2 +- go.sum | 6 ++-- .../storage/fs/ocis/blobstore/blobstore.go | 34 +++++++++++++------ .../utils/decomposedfs/decomposedfs.go | 15 +++++--- .../storage/utils/decomposedfs/tree/tree.go | 1 + .../utils/decomposedfs/upload/session.go | 6 +--- vendor/modules.txt | 2 +- 8 files changed, 46 insertions(+), 26 deletions(-) create mode 100644 changelog/unreleased/fix-upload-postprocessing.md diff --git a/changelog/unreleased/fix-upload-postprocessing.md b/changelog/unreleased/fix-upload-postprocessing.md new file mode 100644 index 0000000000..0a74842b2d --- /dev/null +++ b/changelog/unreleased/fix-upload-postprocessing.md @@ -0,0 +1,6 @@ +Bugfix: Fix the upload postprocessing + +We fixed the upload postprocessing when the destination file does not exist anymore. + +https://github.com/owncloud/ocis/pull/8117 +https://github.com/owncloud/ocis/issues/7909 diff --git a/go.mod b/go.mod index 47aded340c..1de9f44b12 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/coreos/go-oidc v2.2.1+incompatible github.com/coreos/go-oidc/v3 v3.9.0 github.com/cs3org/go-cs3apis v0.0.0-20231023073225-7748710e0781 - github.com/cs3org/reva/v2 v2.18.1-0.20240102110246-e8443f274000 + github.com/cs3org/reva/v2 v2.18.1-0.20240104084554-e85441869c2b github.com/dhowden/tag v0.0.0-20230630033851-978a0926ee25 github.com/disintegration/imaging v1.6.2 github.com/dutchcoders/go-clamd v0.0.0-20170520113014-b970184f4d9e diff --git a/go.sum b/go.sum index 9d43d71dfa..a3c9aa50a8 100644 --- a/go.sum +++ b/go.sum @@ -1017,10 +1017,8 @@ github.com/crewjam/saml v0.4.14 h1:g9FBNx62osKusnFzs3QTN5L9CVA/Egfgm+stJShzw/c= github.com/crewjam/saml v0.4.14/go.mod h1:UVSZCf18jJkk6GpWNVqcyQJMD5HsRugBPf4I1nl2mME= github.com/cs3org/go-cs3apis v0.0.0-20231023073225-7748710e0781 h1:BUdwkIlf8IS2FasrrPg8gGPHQPOrQ18MS1Oew2tmGtY= github.com/cs3org/go-cs3apis v0.0.0-20231023073225-7748710e0781/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY= -github.com/cs3org/reva/v2 v2.18.0 h1:ZfPE+0fgp1ptLIVF+xhMwwQRG1je2OFumMVqjE42AR4= -github.com/cs3org/reva/v2 v2.18.0/go.mod h1:QW31Q1IQ9ZCJMFv3u8/SdHSyLfCcSVNcRbqIJj+Y+7o= -github.com/cs3org/reva/v2 v2.18.1-0.20240102110246-e8443f274000 h1:HO7rqvYYSGLLpOo5s7/TIJjh/6xu6o/VsyUS4sXcuhA= -github.com/cs3org/reva/v2 v2.18.1-0.20240102110246-e8443f274000/go.mod h1:QW31Q1IQ9ZCJMFv3u8/SdHSyLfCcSVNcRbqIJj+Y+7o= +github.com/cs3org/reva/v2 v2.18.1-0.20240104084554-e85441869c2b h1:Nh0SZn2MyCWC/gmV6le7e9eVzux9WWGPQ/nECgh/gyg= +github.com/cs3org/reva/v2 v2.18.1-0.20240104084554-e85441869c2b/go.mod h1:QW31Q1IQ9ZCJMFv3u8/SdHSyLfCcSVNcRbqIJj+Y+7o= github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg= github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= diff --git a/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/ocis/blobstore/blobstore.go b/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/ocis/blobstore/blobstore.go index 3f12d4905b..7afd3eeb55 100644 --- a/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/ocis/blobstore/blobstore.go +++ b/vendor/github.com/cs3org/reva/v2/pkg/storage/fs/ocis/blobstore/blobstore.go @@ -20,6 +20,7 @@ package blobstore import ( "bufio" + "fmt" "io" "os" "path/filepath" @@ -49,8 +50,10 @@ func New(root string) (*Blobstore, error) { // Upload stores some data in the blobstore under the given key func (bs *Blobstore) Upload(node *node.Node, source string) error { - - dest := bs.path(node) + dest, err := bs.path(node) + if err != nil { + return err + } // ensure parent path exists if err := os.MkdirAll(filepath.Dir(dest), 0700); err != nil { return errors.Wrap(err, "Decomposedfs: oCIS blobstore: error creating parent folders for blob") @@ -69,13 +72,13 @@ func (bs *Blobstore) Upload(node *node.Node, source string) error { f, err := os.OpenFile(dest, os.O_CREATE|os.O_WRONLY, 0700) if err != nil { - return errors.Wrapf(err, "could not open blob '%s' for writing", bs.path(node)) + return errors.Wrapf(err, "could not open blob '%s' for writing", dest) } w := bufio.NewWriter(f) _, err = w.ReadFrom(file) if err != nil { - return errors.Wrapf(err, "could not write blob '%s'", bs.path(node)) + return errors.Wrapf(err, "could not write blob '%s'", dest) } return w.Flush() @@ -83,26 +86,37 @@ func (bs *Blobstore) Upload(node *node.Node, source string) error { // Download retrieves a blob from the blobstore for reading func (bs *Blobstore) Download(node *node.Node) (io.ReadCloser, error) { - file, err := os.Open(bs.path(node)) + dest, err := bs.path(node) if err != nil { - return nil, errors.Wrapf(err, "could not read blob '%s'", bs.path(node)) + return nil, err + } + file, err := os.Open(dest) + if err != nil { + return nil, errors.Wrapf(err, "could not read blob '%s'", dest) } return file, nil } // Delete deletes a blob from the blobstore func (bs *Blobstore) Delete(node *node.Node) error { - if err := utils.RemoveItem(bs.path(node)); err != nil { - return errors.Wrapf(err, "could not delete blob '%s'", bs.path(node)) + dest, err := bs.path(node) + if err != nil { + return err + } + if err := utils.RemoveItem(dest); err != nil { + return errors.Wrapf(err, "could not delete blob '%s'", dest) } return nil } -func (bs *Blobstore) path(node *node.Node) string { +func (bs *Blobstore) path(node *node.Node) (string, error) { + if node.BlobID == "" { + return "", fmt.Errorf("blobstore: BlobID is empty") + } return filepath.Join( bs.root, filepath.Clean(filepath.Join( "/", "spaces", lookup.Pathify(node.SpaceID, 1, 2), "blobs", lookup.Pathify(node.BlobID, 4, 2)), ), - ) + ), nil } diff --git a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/decomposedfs.go b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/decomposedfs.go index bca49ef185..7a4e0f173c 100644 --- a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/decomposedfs.go +++ b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/decomposedfs.go @@ -274,16 +274,21 @@ func (fs *Decomposedfs) Postprocessing(ch <-chan events.Event) { continue // NOTE: since we can't get the upload, we can't delete the blob } - var ( - failed bool - keepUpload bool - ) - n, err := session.Node(ctx) if err != nil { log.Error().Err(err).Str("uploadID", ev.UploadID).Msg("could not read node") continue } + if !n.Exists { + log.Debug().Str("uploadID", ev.UploadID).Str("nodeID", session.NodeID()).Msg("node no longer exists") + fs.sessionStore.Cleanup(ctx, session, false, false) + continue + } + + var ( + failed bool + keepUpload bool + ) switch ev.Outcome { default: diff --git a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/tree/tree.go b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/tree/tree.go index e0610ed5d5..136a8fb812 100644 --- a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/tree/tree.go +++ b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/tree/tree.go @@ -375,6 +375,7 @@ func (t *Tree) ListFolder(ctx context.Context, n *node.Node) ([]*node.Node, erro // Spawn workers that'll concurrently work the queue for i := 0; i < numWorkers; i++ { g.Go(func() error { + var err error for name := range work { path := filepath.Join(dir, name) nodeID := getNodeIDFromCache(ctx, path, t.idCache) diff --git a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/upload/session.go b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/upload/session.go index bd9dc085f0..4107ea5e2d 100644 --- a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/upload/session.go +++ b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/upload/session.go @@ -162,11 +162,7 @@ func (s *OcisSession) HeaderIfUnmodifiedSince() string { // Node returns the node for the session func (s *OcisSession) Node(ctx context.Context) (*node.Node, error) { - n, err := node.ReadNode(ctx, s.store.lu, s.SpaceID(), s.info.Storage["NodeId"], false, nil, true) - if err != nil { - return nil, err - } - return n, nil + return node.ReadNode(ctx, s.store.lu, s.SpaceID(), s.info.Storage["NodeId"], false, nil, true) } // ID returns the upload session id diff --git a/vendor/modules.txt b/vendor/modules.txt index 3d3891977a..7cbe6f14c8 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -362,7 +362,7 @@ github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1 github.com/cs3org/go-cs3apis/cs3/storage/registry/v1beta1 github.com/cs3org/go-cs3apis/cs3/tx/v1beta1 github.com/cs3org/go-cs3apis/cs3/types/v1beta1 -# github.com/cs3org/reva/v2 v2.18.1-0.20240102110246-e8443f274000 +# github.com/cs3org/reva/v2 v2.18.1-0.20240104084554-e85441869c2b ## explicit; go 1.21 github.com/cs3org/reva/v2/cmd/revad/internal/grace github.com/cs3org/reva/v2/cmd/revad/runtime