diff --git a/changelog/unreleased/bump-reva.md b/changelog/unreleased/bump-reva.md index 1e32339c10..1160d3471b 100644 --- a/changelog/unreleased/bump-reva.md +++ b/changelog/unreleased/bump-reva.md @@ -2,6 +2,7 @@ Enhancement: Update reva to latest edge version We update reva to the latest edge version to get the latest fixes and features. +https://github.com/owncloud/ocis/pull/8287 https://github.com/owncloud/ocis/pull/8278 https://github.com/owncloud/ocis/pull/8264 https://github.com/owncloud/ocis/pull/8100 diff --git a/changelog/unreleased/postprocessing-bulk-restart.md b/changelog/unreleased/postprocessing-bulk-restart.md new file mode 100644 index 0000000000..d2816f4f5a --- /dev/null +++ b/changelog/unreleased/postprocessing-bulk-restart.md @@ -0,0 +1,5 @@ +Enhancement: Allow restarting multiple uploads with one command + +Allows to restart all commands in a specific state. + +https://github.com/owncloud/ocis/pull/8287 diff --git a/go.mod b/go.mod index 0692f6ef53..5a2b4e45e5 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/coreos/go-oidc v2.2.1+incompatible github.com/coreos/go-oidc/v3 v3.9.0 github.com/cs3org/go-cs3apis v0.0.0-20231023073225-7748710e0781 - github.com/cs3org/reva/v2 v2.18.1-0.20240124094635-6eec406c0be7 + github.com/cs3org/reva/v2 v2.18.1-0.20240126141248-c9e4a3bcd0da github.com/dhowden/tag v0.0.0-20230630033851-978a0926ee25 github.com/disintegration/imaging v1.6.2 github.com/dutchcoders/go-clamd v0.0.0-20170520113014-b970184f4d9e diff --git a/go.sum b/go.sum index 659ada4ed7..5f68995a9c 100644 --- a/go.sum +++ b/go.sum @@ -1018,8 +1018,8 @@ github.com/crewjam/saml v0.4.14 h1:g9FBNx62osKusnFzs3QTN5L9CVA/Egfgm+stJShzw/c= github.com/crewjam/saml v0.4.14/go.mod h1:UVSZCf18jJkk6GpWNVqcyQJMD5HsRugBPf4I1nl2mME= github.com/cs3org/go-cs3apis v0.0.0-20231023073225-7748710e0781 h1:BUdwkIlf8IS2FasrrPg8gGPHQPOrQ18MS1Oew2tmGtY= github.com/cs3org/go-cs3apis v0.0.0-20231023073225-7748710e0781/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY= -github.com/cs3org/reva/v2 v2.18.1-0.20240124094635-6eec406c0be7 h1:g7vQAbo64ziFqqhKcim3JCjDW1zqHy9imAm2HZmmK8w= -github.com/cs3org/reva/v2 v2.18.1-0.20240124094635-6eec406c0be7/go.mod h1:GCN3g6uYE0Nvd31dGlhaGGyUviUfbG2NkecPRv5oSc4= +github.com/cs3org/reva/v2 v2.18.1-0.20240126141248-c9e4a3bcd0da h1:VgWIr/lE6cv2f5IjjWgR0LOAK41gsUytBsSZo/4DRq4= +github.com/cs3org/reva/v2 v2.18.1-0.20240126141248-c9e4a3bcd0da/go.mod h1:GCN3g6uYE0Nvd31dGlhaGGyUviUfbG2NkecPRv5oSc4= github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg= github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= diff --git a/services/postprocessing/README.md b/services/postprocessing/README.md index 282278d04c..9e08ba2180 100644 --- a/services/postprocessing/README.md +++ b/services/postprocessing/README.md @@ -90,3 +90,11 @@ ocis storage-users uploads list ```bash ocis postprocessing restart -u ``` + +Instead of starting one specific upload, a system admin can also restart all uploads that are currently in a specific step. +Examples: +``` +ocis postprocessing restart # Restarts all uploads where postprocessing is finished, but upload is not finished +ocis postprocessing restart -s "finished" # Equivalent to the above +ocis postprocessing restart -s "virusscan" # Restart all uploads currently in virusscan step +``` diff --git a/services/postprocessing/pkg/command/postprocessing.go b/services/postprocessing/pkg/command/postprocessing.go index 99c853a0c9..ee15588963 100644 --- a/services/postprocessing/pkg/command/postprocessing.go +++ b/services/postprocessing/pkg/command/postprocessing.go @@ -2,7 +2,6 @@ package command import ( "context" - "time" "github.com/cs3org/reva/v2/pkg/events" "github.com/cs3org/reva/v2/pkg/events/stream" @@ -20,10 +19,15 @@ func RestartPostprocessing(cfg *config.Config) *cli.Command { Usage: "restart postprocessing for an uploadID", Flags: []cli.Flag{ &cli.StringFlag{ - Name: "upload-id", - Aliases: []string{"u"}, - Required: true, - Usage: "the uploadid to restart", + Name: "upload-id", + Aliases: []string{"u"}, + Usage: "the uploadid to restart. Ignored if unset.", + }, + &cli.StringFlag{ + Name: "step", + Aliases: []string{"s"}, + Usage: "restarts all uploads in the given postprocessing step. Ignored if upload-id is set.", + Value: "finished", // Calling `ocis postprocessing restart` without any arguments will restart all uploads that are finished but failed to move the uploed from the upload area to the blobstore. }, }, Before: func(c *cli.Context) error { @@ -35,24 +39,18 @@ func RestartPostprocessing(cfg *config.Config) *cli.Command { return err } + uid, step := c.String("upload-id"), "" + if uid == "" { + step = c.String("step") + } + ev := events.ResumePostprocessing{ - UploadID: c.String("upload-id"), + UploadID: uid, + Step: events.Postprocessingstep(step), Timestamp: utils.TSNow(), } - if err := events.Publish(context.Background(), stream, ev); err != nil { - return err - } - - // go-micro nats implementation uses async publishing, - // therefore we need to manually wait. - // - // FIXME: upstream pr - // - // https://github.com/go-micro/plugins/blob/3e77393890683be4bacfb613bc5751867d584692/v4/events/natsjs/nats.go#L115 - time.Sleep(5 * time.Second) - - return nil + return events.Publish(context.Background(), stream, ev) }, } } diff --git a/services/postprocessing/pkg/postprocessing/postprocessing.go b/services/postprocessing/pkg/postprocessing/postprocessing.go index b2764f43f9..e3c9ca6bff 100644 --- a/services/postprocessing/pkg/postprocessing/postprocessing.go +++ b/services/postprocessing/pkg/postprocessing/postprocessing.go @@ -65,14 +65,14 @@ func (pp *Postprocessing) NextStep(ev events.PostprocessingStepFinished) interfa // CurrentStep returns the current postprocessing step func (pp *Postprocessing) CurrentStep() interface{} { - if pp.Status.Outcome != "" { + if pp.Status.CurrentStep == events.PPStepFinished { return pp.finished(pp.Status.Outcome) } return pp.step(pp.Status.CurrentStep) } // Delay will sleep the configured time then continue -func (pp *Postprocessing) Delay(ev events.StartPostprocessingStep) interface{} { +func (pp *Postprocessing) Delay() interface{} { time.Sleep(pp.config.Delayprocessing) return pp.next(events.PPStepDelay) } @@ -106,6 +106,7 @@ func (pp *Postprocessing) step(next events.Postprocessingstep) events.StartPostp } func (pp *Postprocessing) finished(outcome events.PostprocessingOutcome) events.PostprocessingFinished { + pp.Status.CurrentStep = events.PPStepFinished pp.Status.Outcome = outcome return events.PostprocessingFinished{ UploadID: pp.ID, diff --git a/services/postprocessing/pkg/service/service.go b/services/postprocessing/pkg/service/service.go index 50355001b1..f2a3c122e2 100644 --- a/services/postprocessing/pkg/service/service.go +++ b/services/postprocessing/pkg/service/service.go @@ -8,6 +8,7 @@ import ( "time" "github.com/cs3org/reva/v2/pkg/events" + "github.com/cs3org/reva/v2/pkg/utils" "github.com/owncloud/ocis/v2/ocis-pkg/log" "github.com/owncloud/ocis/v2/services/postprocessing/pkg/config" "github.com/owncloud/ocis/v2/services/postprocessing/pkg/postprocessing" @@ -142,29 +143,20 @@ func (pps *PostprocessingService) processEvent(e events.Event) error { pps.log.Error().Str("uploadID", ev.UploadID).Err(err).Msg("cannot get upload") return fmt.Errorf("%w: cannot get upload", errEvent) } - next = pp.Delay(ev) + next = pp.Delay() case events.UploadReady: + if ev.Failed { + // the upload failed - let's keep it around for a while + return nil + } + // the storage provider thinks the upload is done - so no need to keep it any more if err := pps.store.Delete(ev.UploadID); err != nil { pps.log.Error().Str("uploadID", ev.UploadID).Err(err).Msg("cannot delete upload") return fmt.Errorf("%w: cannot delete upload", errEvent) } case events.ResumePostprocessing: - pp, err = pps.getPP(pps.store, ev.UploadID) - if err != nil { - if err == store.ErrNotFound { - if err := events.Publish(ctx, pps.pub, events.RestartPostprocessing{ - UploadID: ev.UploadID, - Timestamp: ev.Timestamp, - }); err != nil { - pps.log.Error().Str("uploadID", ev.UploadID).Err(err).Msg("cannot publish RestartPostprocessing event") - } - return fmt.Errorf("%w: cannot publish RestartPostprocessing event", errEvent) - } - pps.log.Error().Str("uploadID", ev.UploadID).Err(err).Msg("cannot get upload") - return fmt.Errorf("%w: cannot get upload", errEvent) - } - next = pp.CurrentStep() + return pps.handleResumePPEvent(ctx, ev) } if pp != nil { @@ -182,30 +174,6 @@ func (pps *PostprocessingService) processEvent(e events.Event) error { return nil } -func getSteps(c config.Postprocessing) []events.Postprocessingstep { - // NOTE: improved version only allows configuring order of postprocessing steps - // But we aim for a system where postprocessing steps can be configured per space, ideally by the spaceadmin itself - // We need to iterate over configuring PP service when we see fit - var steps []events.Postprocessingstep - for _, s := range c.Steps { - steps = append(steps, events.Postprocessingstep(s)) - } - - return steps -} - -func storePP(sto store.Store, pp *postprocessing.Postprocessing) error { - b, err := json.Marshal(pp) - if err != nil { - return err - } - - return sto.Write(&store.Record{ - Key: pp.ID, - Value: b, - }) -} - func (pps *PostprocessingService) getPP(sto store.Store, uploadID string) (*postprocessing.Postprocessing, error) { recs, err := sto.Read(uploadID) if err != nil { @@ -224,3 +192,95 @@ func (pps *PostprocessingService) getPP(sto store.Store, uploadID string) (*post return pp, nil } + +func getSteps(c config.Postprocessing) []events.Postprocessingstep { + // NOTE: improved version only allows configuring order of postprocessing steps + // But we aim for a system where postprocessing steps can be configured per space, ideally by the spaceadmin itself + // We need to iterate over configuring PP service when we see fit + steps := make([]events.Postprocessingstep, 0, len(c.Steps)) + for _, s := range c.Steps { + steps = append(steps, events.Postprocessingstep(s)) + } + + return steps +} + +func storePP(sto store.Store, pp *postprocessing.Postprocessing) error { + b, err := json.Marshal(pp) + if err != nil { + return err + } + + return sto.Write(&store.Record{ + Key: pp.ID, + Value: b, + }) +} + +func (pps *PostprocessingService) handleResumePPEvent(ctx context.Context, ev events.ResumePostprocessing) error { + ids := []string{ev.UploadID} + if ev.Step != "" { + ids = pps.findUploadsByStep(ev.Step) + } + + for _, id := range ids { + if err := pps.resumePP(ctx, id); err != nil { + pps.log.Error().Str("uploadID", id).Err(err).Msg("cannot resume upload") + return fmt.Errorf("%w: cannot resume upload", errEvent) + } + } + return nil +} + +func (pps *PostprocessingService) resumePP(ctx context.Context, uploadID string) error { + pp, err := pps.getPP(pps.store, uploadID) + if err != nil { + if err == store.ErrNotFound { + if err := events.Publish(ctx, pps.pub, events.RestartPostprocessing{ + UploadID: uploadID, + Timestamp: utils.TSNow(), + }); err != nil { + return err + } + return nil + } + return fmt.Errorf("%w: cannot get upload", errEvent) + } + + return events.Publish(ctx, pps.pub, pp.CurrentStep()) +} + +func (pps *PostprocessingService) findUploadsByStep(step events.Postprocessingstep) []string { + var ids []string + + keys, err := pps.store.List() + if err != nil { + pps.log.Error().Err(err).Msg("cannot list uploads") + } + + for _, k := range keys { + rec, err := pps.store.Read(k) + if err != nil { + pps.log.Error().Err(err).Msg("cannot read upload") + continue + } + + if len(rec) != 1 { + pps.log.Error().Err(err).Msg("expected only one result") + continue + } + + pp := &postprocessing.Postprocessing{} + err = json.Unmarshal(rec[0].Value, pp) + if err != nil { + pps.log.Error().Err(err).Msg("cannot unmarshal upload") + continue + } + + if pp.Status.CurrentStep == step { + ids = append(ids, pp.ID) + } + } + + return ids +} diff --git a/tests/acceptance/features/apiSpacesDavOperation/moveByFileId.feature b/tests/acceptance/features/apiSpacesDavOperation/moveByFileId.feature index 0a92b25b02..7d46357de8 100644 --- a/tests/acceptance/features/apiSpacesDavOperation/moveByFileId.feature +++ b/tests/acceptance/features/apiSpacesDavOperation/moveByFileId.feature @@ -94,8 +94,8 @@ Feature: moving/renaming file using file id And user "Brian" has uploaded file with content "some data" to "/test.txt" And we save it into "FILEID" When user "Brian" moves a file "test.txt" into "folder" inside space "Shares" using file-id path "" - Then the HTTP status code should be "403" - And the value of the item "/d:error/s:message" in the response about user "Brian" should be "cross storage moves are not permitted, use copy and delete" + Then the HTTP status code should be "502" + And the value of the item "/d:error/s:message" in the response about user "Brian" should be "cross storage moves are not supported, use copy and delete" And for user "Brian" folder "/" of the space "Personal" should contain these files: | test.txt | But for user "Alice" folder "folder" of the space "Personal" should not contain these files: @@ -351,7 +351,7 @@ Feature: moving/renaming file using file id And user "Alice" has created folder "testshare" And user "Alice" has shared folder "testshare" with user "Brian" with permissions "" When user "Brian" moves a file "textfile.txt" into "testshare" inside space "Shares" using file-id path "" - Then the HTTP status code should be "403" + Then the HTTP status code should be "502" And for user "Brian" folder "/" of the space "project-space" should contain these files: | textfile.txt | But for user "Brian" folder "testshare" of the space "Shares" should not contain these files: @@ -475,7 +475,7 @@ Feature: moving/renaming file using file id And we save it into "FILEID" And user "Alice" has shared folder "folder" with user "Brian" with permissions "read" When user "Brian" moves a file "Shares/folder/test.txt" into "folder/sub-folder" inside space "Shares" using file-id path "" - Then the HTTP status code should be "403" + Then the HTTP status code should be "502" And for user "Brian" folder "folder/sub-folder" of the space "Shares" should not contain these files: | test.txt | And for user "Alice" folder "folder/sub-folder" of the space "Personal" should not contain these files: @@ -499,7 +499,7 @@ Feature: moving/renaming file using file id And user "Alice" has shared folder "testshare1" with user "Brian" with permissions "" And user "Alice" has shared folder "testshare2" with user "Brian" with permissions "" When user "Brian" moves a file "Shares/testshare1/textfile.txt" into "testshare2" inside space "Shares" using file-id path "" - Then the HTTP status code should be "403" + Then the HTTP status code should be "502" And for user "Brian" folder "testshare1" of the space "Shares" should contain these files: | textfile.txt | But for user "Brian" folder "testshare2" of the space "Shares" should not contain these files: @@ -697,7 +697,7 @@ Feature: moving/renaming file using file id And we save it into "FILEID" And user "Alice" has shared folder "/folder" with user "Brian" with permissions "read" When user "Brian" renames a file "Shares/folder/test.txt" into "folder/sub-folder/renamed.txt" inside space "Shares" using file-id path "" - Then the HTTP status code should be "403" + Then the HTTP status code should be "502" And for user "Brian" folder "folder" of the space "Shares" should contain these files: | test.txt | But for user "Brian" folder "folder/sub-folder" of the space "Shares" should not contain these files: diff --git a/tests/acceptance/features/apiSpacesShares/moveSpaces.feature b/tests/acceptance/features/apiSpacesShares/moveSpaces.feature index 47130d389d..20413c713e 100644 --- a/tests/acceptance/features/apiSpacesShares/moveSpaces.feature +++ b/tests/acceptance/features/apiSpacesShares/moveSpaces.feature @@ -125,7 +125,7 @@ Feature: move (rename) file | role | | And user "Brian" has shared folder "/testshare" with user "Alice" with permissions "" When user "Alice" moves file "project.txt" from space "Project" to "/testshare/project.txt" inside space "Shares" using the WebDAV API - Then the HTTP status code should be "403" + Then the HTTP status code should be "502" And for user "Alice" the space "Project" should contain these entries: | project.txt | But for user "Alice" folder "testshare" of the space "Shares" should not contain these entries: @@ -168,7 +168,7 @@ Feature: move (rename) file And user "Brian" has shared folder "/testshare" with user "Alice" with permissions "" And user "Alice" has uploaded file with content "personal content" to "personal.txt" When user "Alice" moves file "personal.txt" from space "Personal" to "/testshare/personal.txt" inside space "Shares" using the WebDAV API - Then the HTTP status code should be "403" + Then the HTTP status code should be "502" And for user "Alice" the space "Personal" should contain these entries: | personal.txt | But for user "Alice" folder "testshare" of the space "Shares" should not contain these entries: @@ -185,7 +185,7 @@ Feature: move (rename) file And user "Brian" has uploaded file with content "testshare content" to "/testshare/testshare.txt" And user "Brian" has shared folder "/testshare" with user "Alice" with permissions "" When user "Alice" moves file "/testshare/testshare.txt" from space "Shares" to "testshare.txt" inside space "Personal" using the WebDAV API - Then the HTTP status code should be "403" + Then the HTTP status code should be "502" And for user "Alice" the space "Personal" should not contain these entries: | testshare.txt | And for user "Alice" folder "testshare" of the space "Shares" should contain these entries: @@ -207,7 +207,7 @@ Feature: move (rename) file And user "Brian" has uploaded file with content "testshare content" to "/testshare/testshare.txt" And user "Brian" has shared folder "/testshare" with user "Alice" with permissions "" When user "Alice" moves file "/testshare/testshare.txt" from space "Shares" to "testshare.txt" inside space "Project" using the WebDAV API - Then the HTTP status code should be "403" + Then the HTTP status code should be "502" And for user "Alice" the space "Project" should not contain these entries: | /testshare.txt | And for user "Alice" folder "testshare" of the space "Shares" should contain these entries: @@ -232,7 +232,7 @@ Feature: move (rename) file And user "Brian" has shared folder "/testshare1" with user "Alice" with permissions "" And user "Brian" has shared folder "/testshare2" with user "Alice" with permissions "" When user "Alice" moves file "/testshare1/testshare1.txt" from space "Shares" to "/testshare2/testshare1.txt" inside space "Shares" using the WebDAV API - Then the HTTP status code should be "403" + Then the HTTP status code should be "502" And for user "Alice" folder "testshare1" of the space "Shares" should contain these entries: | testshare1.txt | But for user "Alice" folder "testshare2" of the space "Shares" should not contain these entries: diff --git a/tests/acceptance/features/coreApiShareManagementToShares/moveShareInsideAnotherShare.feature b/tests/acceptance/features/coreApiShareManagementToShares/moveShareInsideAnotherShare.feature index b13cbd7e17..0c16fe0f04 100644 --- a/tests/acceptance/features/coreApiShareManagementToShares/moveShareInsideAnotherShare.feature +++ b/tests/acceptance/features/coreApiShareManagementToShares/moveShareInsideAnotherShare.feature @@ -20,7 +20,7 @@ Feature: moving a share inside another share Scenario: share receiver cannot move a whole share inside another share When user "Brian" moves folder "Shares/folderB" to "Shares/folderA/folderB" using the WebDAV API - Then the HTTP status code should be "403" + Then the HTTP status code should be "502" And as "Alice" folder "/folderB" should exist And as "Brian" folder "/Shares/folderB" should exist And as "Alice" file "/folderB/fileB.txt" should exist @@ -43,7 +43,7 @@ Feature: moving a share inside another share And user "Brian" has created folder "localFolder/subFolder" And user "Brian" has uploaded file with content "local text" to "/localFolder/localFile.txt" When user "Brian" moves folder "localFolder" to "Shares/folderA/localFolder" using the WebDAV API - Then the HTTP status code should be "403" + Then the HTTP status code should be "502" And as "Brian" folder "/Shares/folderA/localFolder" should not exist And as "Alice" folder "/folderA/localFolder" should not exist And as "Brian" folder "/localFolder" should exist @@ -52,8 +52,7 @@ Feature: moving a share inside another share Scenario: share receiver tries to move a whole share inside a local folder Given user "Brian" has created folder "localFolder" And user "Brian" has uploaded file with content "local text" to "/localFolder/localFile.txt" - # On oCIS you cannot move received shares out of the "Shares" folder When user "Brian" moves folder "Shares/folderB" to "localFolder/folderB" using the WebDAV API - Then the HTTP status code should be "403" + Then the HTTP status code should be "502" And as "Alice" file "/folderB/fileB.txt" should exist And as "Brian" file "/Shares/folderB/fileB.txt" should exist diff --git a/tests/acceptance/features/coreApiShareOperationsToShares1/changingFilesShare.feature b/tests/acceptance/features/coreApiShareOperationsToShares1/changingFilesShare.feature index 54f5cf2c2a..0aea127ba2 100644 --- a/tests/acceptance/features/coreApiShareOperationsToShares1/changingFilesShare.feature +++ b/tests/acceptance/features/coreApiShareOperationsToShares1/changingFilesShare.feature @@ -20,7 +20,7 @@ Feature: sharing And user "Alice" has shared folder "/share1" with user "Brian" And user "Alice" has shared folder "/share2" with user "Brian" When user "Brian" moves file "/Shares/share1/textfile0.txt" to "/Shares/share2/textfile0.txt" using the WebDAV API - Then the HTTP status code should be "403" + Then the HTTP status code should be "502" And as "Brian" file "/Shares/share1/textfile0.txt" should exist And as "Alice" file "share1/textfile0.txt" should exist But as "Brian" file "/Shares/share2/textfile0.txt" should not exist diff --git a/tests/acceptance/features/coreApiWebdavMove2/moveShareOnOcis.feature b/tests/acceptance/features/coreApiWebdavMove2/moveShareOnOcis.feature index feba1ab4f6..da4e86296d 100644 --- a/tests/acceptance/features/coreApiWebdavMove2/moveShareOnOcis.feature +++ b/tests/acceptance/features/coreApiWebdavMove2/moveShareOnOcis.feature @@ -45,7 +45,7 @@ Feature: move (rename) file | shareWith | Alice | And user "Alice" has uploaded file with content "test data" to "/testfile.txt" When user "Alice" moves file "/testfile.txt" to "Shares/testshare/testfile.txt" using the WebDAV API - Then the HTTP status code should be "403" + Then the HTTP status code should be "502" And as "Alice" file "Shares/testshare/testfile.txt" should not exist And as "Brian" file "testshare/testfile.txt" should not exist But as "Alice" file "/testfile.txt" should exist @@ -95,7 +95,7 @@ Feature: move (rename) file | permissions | | | shareWith | Alice | When user "Alice" moves file "/Shares/testshare/testfile.txt" to "/testfile.txt" using the WebDAV API - Then the HTTP status code should be "403" + Then the HTTP status code should be "502" And as "Alice" file "/Shares/testshare/testfile.txt" should exist And as "Brian" file "/testshare/testfile.txt" should exist Examples: @@ -146,7 +146,7 @@ Feature: move (rename) file And user "Alice" has created folder "/testsubfolder" And user "Alice" has uploaded file with content "test data" to "/testsubfolder/testfile.txt" When user "Alice" moves folder "/testsubfolder" to "Shares/testshare/testsubfolder" using the WebDAV API - Then the HTTP status code should be "403" + Then the HTTP status code should be "502" And as "Alice" folder "/Shares/testshare/testsubfolder" should not exist And as "Brian" folder "/testshare/testsubfolder" should not exist But as "Alice" folder "/testsubfolder" should exist @@ -202,7 +202,7 @@ Feature: move (rename) file | permissions | | | shareWith | Alice | When user "Alice" moves folder "/Shares/testshare/testsubfolder" to "/testsubfolder" using the WebDAV API - Then the HTTP status code should be "403" + Then the HTTP status code should be "502" And as "Alice" folder "/Shares/testshare/testsubfolder" should exist And as "Brian" folder "/testshare/testsubfolder" should exist Examples: diff --git a/vendor/github.com/cs3org/reva/v2/internal/grpc/services/gateway/storageprovider.go b/vendor/github.com/cs3org/reva/v2/internal/grpc/services/gateway/storageprovider.go index 6c1ac74245..cf3a7387e3 100644 --- a/vendor/github.com/cs3org/reva/v2/internal/grpc/services/gateway/storageprovider.go +++ b/vendor/github.com/cs3org/reva/v2/internal/grpc/services/gateway/storageprovider.go @@ -709,7 +709,7 @@ func (s *svc) Move(ctx context.Context, req *provider.MoveRequest) (*provider.Mo if sourceProviderInfo.Address != destProviderInfo.Address { return &provider.MoveResponse{ - Status: status.NewPermissionDenied(ctx, nil, "cross storage moves are not permitted, use copy and delete"), + Status: status.NewUnimplemented(ctx, nil, "cross storage moves are not supported, use copy and delete"), }, nil } diff --git a/vendor/github.com/cs3org/reva/v2/internal/grpc/services/publicshareprovider/publicshareprovider.go b/vendor/github.com/cs3org/reva/v2/internal/grpc/services/publicshareprovider/publicshareprovider.go index 201b2c0f97..3df749f882 100644 --- a/vendor/github.com/cs3org/reva/v2/internal/grpc/services/publicshareprovider/publicshareprovider.go +++ b/vendor/github.com/cs3org/reva/v2/internal/grpc/services/publicshareprovider/publicshareprovider.go @@ -68,7 +68,6 @@ type config struct { } type passwordPolicy struct { - Disabled bool `mapstructure:"disabled"` MinCharacters int `mapstructure:"min_characters"` MinLowerCaseCharacters int `mapstructure:"min_lowercase_characters"` MinUpperCaseCharacters int `mapstructure:"min_uppercase_characters"` @@ -174,10 +173,9 @@ func New(m map[string]interface{}, ss *grpc.Server) (rgrpc.Service, error) { func newPasswordPolicy(c *passwordPolicy) password.Validator { if c == nil { - return password.NewPasswordPolicy(true, 0, 0, 0, 0, 0, nil) + return password.NewPasswordPolicy(0, 0, 0, 0, 0, nil) } return password.NewPasswordPolicy( - c.Disabled, c.MinCharacters, c.MinLowerCaseCharacters, c.MinUpperCaseCharacters, diff --git a/vendor/github.com/cs3org/reva/v2/internal/grpc/services/sharesstorageprovider/sharesstorageprovider.go b/vendor/github.com/cs3org/reva/v2/internal/grpc/services/sharesstorageprovider/sharesstorageprovider.go index 1d678d06bc..da3624fd7a 100644 --- a/vendor/github.com/cs3org/reva/v2/internal/grpc/services/sharesstorageprovider/sharesstorageprovider.go +++ b/vendor/github.com/cs3org/reva/v2/internal/grpc/services/sharesstorageprovider/sharesstorageprovider.go @@ -667,7 +667,7 @@ func (s *service) Move(ctx context.Context, req *provider.MoveRequest) (*provide if dstReceivedShare.Share.Id.OpaqueId != srcReceivedShare.Share.Id.OpaqueId { return &provider.MoveResponse{ - Status: status.NewPermissionDenied(ctx, nil, "cross storage moves are not permitted, use copy and delete"), + Status: status.NewUnimplemented(ctx, nil, "cross storage moves are not supported, use copy and delete"), }, nil } diff --git a/vendor/github.com/cs3org/reva/v2/internal/http/services/appprovider/appprovider.go b/vendor/github.com/cs3org/reva/v2/internal/http/services/appprovider/appprovider.go index c121657885..2181e98b95 100644 --- a/vendor/github.com/cs3org/reva/v2/internal/http/services/appprovider/appprovider.go +++ b/vendor/github.com/cs3org/reva/v2/internal/http/services/appprovider/appprovider.go @@ -276,9 +276,8 @@ func (s *svc) handleNew(w http.ResponseWriter, r *http.Request) { return } defer httpRes.Body.Close() - if httpRes.StatusCode == http.StatusForbidden { + if httpRes.StatusCode == http.StatusBadRequest { // the file upload was already finished since it is a zero byte file - // TODO: why do we get a 401 then!? } else if httpRes.StatusCode != http.StatusOK { writeError(w, r, appErrorServerError, "failed to create the file", nil) return diff --git a/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocs/data/capabilities.go b/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocs/data/capabilities.go index f6abda35b0..369ca23179 100644 --- a/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocs/data/capabilities.go +++ b/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocs/data/capabilities.go @@ -117,7 +117,6 @@ type CapabilitiesGraph struct { // CapabilitiesPasswordPolicy hold the password policy capabilities type CapabilitiesPasswordPolicy struct { - Disabled bool `json:"disabled" xml:"disabled" mapstructure:"disabled"` MinCharacters int `json:"min_characters" xml:"min_characters" mapstructure:"min_characters"` MaxCharacters int `json:"max_characters" xml:"max_characters" mapstructure:"max_characters"` MinLowerCaseCharacters int `json:"min_lowercase_characters" xml:"min_lowercase_characters" mapstructure:"min_lowercase_characters"` diff --git a/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/shares.go b/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/shares.go index f75a30b340..59c5ed2f47 100644 --- a/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/shares.go +++ b/vendor/github.com/cs3org/reva/v2/internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/shares.go @@ -1714,10 +1714,9 @@ func publicPwdEnforced(c *config.Config) passwordEnforced { func passwordPolicies(c *config.Config) password.Validator { if c.Capabilities.Capabilities == nil || c.Capabilities.Capabilities.PasswordPolicy == nil { - return password.NewPasswordPolicy(true, 0, 0, 0, 0, 0, nil) + return password.NewPasswordPolicy(0, 0, 0, 0, 0, nil) } return password.NewPasswordPolicy( - c.Capabilities.Capabilities.PasswordPolicy.Disabled, c.Capabilities.Capabilities.PasswordPolicy.MinCharacters, c.Capabilities.Capabilities.PasswordPolicy.MinLowerCaseCharacters, c.Capabilities.Capabilities.PasswordPolicy.MinUpperCaseCharacters, diff --git a/vendor/github.com/cs3org/reva/v2/pkg/events/postprocessing.go b/vendor/github.com/cs3org/reva/v2/pkg/events/postprocessing.go index cd52a6356d..69e3f14ac7 100644 --- a/vendor/github.com/cs3org/reva/v2/pkg/events/postprocessing.go +++ b/vendor/github.com/cs3org/reva/v2/pkg/events/postprocessing.go @@ -42,6 +42,8 @@ var ( PPStepPolicies Postprocessingstep = "policies" // PPStepDelay is the step that processing. Useful for testing or user annoyment PPStepDelay Postprocessingstep = "delay" + // PPStepFinished is the step that signals that postprocessing is finished, but storage provider hasn't acknowledged it yet + PPStepFinished Postprocessingstep = "finished" // PPOutcomeDelete means that the file and the upload should be deleted PPOutcomeDelete PostprocessingOutcome = "delete" @@ -193,6 +195,7 @@ func (UploadReady) Unmarshal(v []byte) (interface{}, error) { // ResumePostprocessing can be emitted to repair broken postprocessing type ResumePostprocessing struct { UploadID string + Step Postprocessingstep Timestamp *types.Timestamp } diff --git a/vendor/github.com/cs3org/reva/v2/pkg/password/password_policies.go b/vendor/github.com/cs3org/reva/v2/pkg/password/password_policies.go index ac126c9e87..ec98783d31 100644 --- a/vendor/github.com/cs3org/reva/v2/pkg/password/password_policies.go +++ b/vendor/github.com/cs3org/reva/v2/pkg/password/password_policies.go @@ -18,7 +18,6 @@ type Validator interface { // Policies represents a password validation rules type Policies struct { - disabled bool minCharacters int minLowerCaseCharacters int minUpperCaseCharacters int @@ -30,9 +29,8 @@ type Policies struct { } // NewPasswordPolicy returns a new NewPasswordPolicy instance -func NewPasswordPolicy(disabled bool, minCharacters, minLowerCaseCharacters, minUpperCaseCharacters, minDigits, minSpecialCharacters int, bannedPasswordsList map[string]struct{}) Validator { +func NewPasswordPolicy(minCharacters, minLowerCaseCharacters, minUpperCaseCharacters, minDigits, minSpecialCharacters int, bannedPasswordsList map[string]struct{}) Validator { p := &Policies{ - disabled: disabled, minCharacters: minCharacters, minLowerCaseCharacters: minLowerCaseCharacters, minUpperCaseCharacters: minUpperCaseCharacters, @@ -48,9 +46,6 @@ func NewPasswordPolicy(disabled bool, minCharacters, minLowerCaseCharacters, min // Validate implements a password validation regarding the policy func (s Policies) Validate(str string) error { - if s.disabled { - return nil - } var allErr error if !utf8.ValidString(str) { return fmt.Errorf("the password contains invalid characters") diff --git a/vendor/github.com/cs3org/reva/v2/pkg/rhttp/datatx/manager/simple/simple.go b/vendor/github.com/cs3org/reva/v2/pkg/rhttp/datatx/manager/simple/simple.go index a89ce71ecd..6699c9f77a 100644 --- a/vendor/github.com/cs3org/reva/v2/pkg/rhttp/datatx/manager/simple/simple.go +++ b/vendor/github.com/cs3org/reva/v2/pkg/rhttp/datatx/manager/simple/simple.go @@ -93,6 +93,13 @@ func (m *manager) Handler(fs storage.FS) (http.Handler, error) { defer func() { metrics.UploadsActive.Sub(1) }() + + if r.ContentLength == 0 { + sublog.Info().Msg("received invalid 0-byte PUT request") + w.WriteHeader(http.StatusBadRequest) + return + } + fn := r.URL.Path defer r.Body.Close() diff --git a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/decomposedfs.go b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/decomposedfs.go index 6e709dfad7..bedb629eeb 100644 --- a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/decomposedfs.go +++ b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/decomposedfs.go @@ -286,8 +286,9 @@ func (fs *Decomposedfs) Postprocessing(ch <-chan events.Event) { } var ( - failed bool - keepUpload bool + failed bool + revertNodeMetadata bool + keepUpload bool ) unmarkPostprocessing := true @@ -297,12 +298,14 @@ func (fs *Decomposedfs) Postprocessing(ch <-chan events.Event) { fallthrough case events.PPOutcomeAbort: failed = true + revertNodeMetadata = true keepUpload = true metrics.UploadSessionsAborted.Inc() case events.PPOutcomeContinue: if err := session.Finalize(); err != nil { log.Error().Err(err).Str("uploadID", ev.UploadID).Msg("could not finalize upload") failed = true + revertNodeMetadata = false keepUpload = true // keep postprocessing status so the upload is not deleted during housekeeping unmarkPostprocessing = false @@ -311,6 +314,7 @@ func (fs *Decomposedfs) Postprocessing(ch <-chan events.Event) { } case events.PPOutcomeDelete: failed = true + revertNodeMetadata = true metrics.UploadSessionsDeleted.Inc() } @@ -337,7 +341,7 @@ func (fs *Decomposedfs) Postprocessing(ch <-chan events.Event) { } } - fs.sessionStore.Cleanup(ctx, session, failed, keepUpload, unmarkPostprocessing) + fs.sessionStore.Cleanup(ctx, session, revertNodeMetadata, keepUpload, unmarkPostprocessing) // remove cache entry in gateway fs.cache.RemoveStatContext(ctx, ev.ExecutingUser.GetId(), &provider.ResourceId{SpaceId: n.SpaceID, OpaqueId: n.ID}) diff --git a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/upload/session.go b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/upload/session.go index 4107ea5e2d..f438d684f9 100644 --- a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/upload/session.go +++ b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/upload/session.go @@ -295,13 +295,9 @@ func (s *OcisSession) MTime() time.Time { return t } -// IsProcessing returns true if the node has entered postprocessing state +// IsProcessing returns true if all bytes have been received. The session then has entered postprocessing state. func (s *OcisSession) IsProcessing() bool { - n, err := s.Node(context.Background()) - if err != nil { - return false - } - return n.IsProcessing(context.Background()) + return s.info.Size == s.info.Offset } // binPath returns the path to the file storing the binary data. diff --git a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/upload/upload.go b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/upload/upload.go index 64c3cb047c..2f2713b454 100644 --- a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/upload/upload.go +++ b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/upload/upload.go @@ -191,7 +191,7 @@ func (session *OcisSession) FinishUpload(ctx context.Context) error { n, err := session.store.CreateNodeForUpload(session, attrs) if err != nil { - session.store.Cleanup(ctx, session, true, false, true) + session.store.Cleanup(ctx, session, true, false, false) return err } diff --git a/vendor/go-micro.dev/v4/registry/cache/cache.go b/vendor/go-micro.dev/v4/registry/cache/cache.go index 5d17d02040..9a7b5e09cd 100644 --- a/vendor/go-micro.dev/v4/registry/cache/cache.go +++ b/vendor/go-micro.dev/v4/registry/cache/cache.go @@ -459,7 +459,7 @@ func (c *cache) Stop() { } func (c *cache) String() string { - return "cache" + return "cached" } // New returns a new cache diff --git a/vendor/modules.txt b/vendor/modules.txt index fdca12de4d..bb6da18d06 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -362,7 +362,7 @@ github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1 github.com/cs3org/go-cs3apis/cs3/storage/registry/v1beta1 github.com/cs3org/go-cs3apis/cs3/tx/v1beta1 github.com/cs3org/go-cs3apis/cs3/types/v1beta1 -# github.com/cs3org/reva/v2 v2.18.1-0.20240124094635-6eec406c0be7 +# github.com/cs3org/reva/v2 v2.18.1-0.20240126141248-c9e4a3bcd0da ## explicit; go 1.21 github.com/cs3org/reva/v2/cmd/revad/internal/grace github.com/cs3org/reva/v2/cmd/revad/runtime