diff --git a/services/storage-users/README.md b/services/storage-users/README.md index e89cbd30a..bee130106 100644 --- a/services/storage-users/README.md +++ b/services/storage-users/README.md @@ -36,10 +36,13 @@ When using Infinite Scale as user storage, a directory named `storage/users/uplo Example cases for expired uploads -* When a user uploads a big file but the file exceeds the user-quota, the upload can't be moved to the target after it has finished. The file stays at the upload location until it is manually cleared. +* In the final step the upload blob is moved from the upload area to the final blobstore (e.g. S3). + * If the bandwidth is limited and the file to transfer can't be transferred completely before the upload expiration time is reached, the file expires and can't be processed. -There are two commands available to manage unfinished uploads +The admin can restart the postprocessing for this with the postprocessing cli. + +The storage users service can only list and clean upload sessions: ```bash ocis storage-users uploads @@ -47,21 +50,38 @@ ocis storage-users uploads ```plaintext COMMANDS: - list Print a list of all incomplete uploads - clean Clean up leftovers from expired uploads + sessions Print a list of upload sessions + clean Clean up leftovers from expired uploads + list Print a list of all incomplete uploads (deprecated) ``` #### Command Examples -Command to identify incomplete uploads +Command to list ongoing upload sessions ```bash -ocis storage-users uploads list +ocis storage-users sessions --expired=false ``` ```plaintext -Incomplete uploads: - - 455bd640-cd08-46e8-a5a0-9304908bd40a (file_example_PPT_1MB.ppt, Size: 1028608, Expires: 2022-08-17T12:35:34+02:00) +Not expired sessions: ++--------------------------------------+--------------------------------------+---------+--------+------+--------------------------------------+--------------------------------------+---------------------------+------------+ +| Space | Upload Id | Name | Offset | Size | Executant | Owner | Expires | Processing | ++--------------------------------------+--------------------------------------+---------+--------+------+--------------------------------------+--------------------------------------+---------------------------+------------+ +| f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c | 5e387954-7313-4223-a904-bf996da6ec0b | foo.txt | 0 | 1234 | f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c | f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c | 2024-01-26T13:04:31+01:00 | false | +| f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c | f066244d-97b2-48e7-a30d-b40fcb60cec6 | bar.txt | 0 | 4321 | f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c | f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c | 2024-01-26T13:18:47+01:00 | false | ++--------------------------------------+--------------------------------------+---------+--------+------+--------------------------------------+--------------------------------------+---------------------------+------------+ +``` + +The sessions command can also output json + +```bash +ocis storage-users sessions --expired=false --json +``` + +```json +{"id":"5e387954-7313-4223-a904-bf996da6ec0b","space":"f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c","filename":"foo.txt","offset":0,"size":1234,"executant":{"idp":"https://cloud.ocis.test","opaque_id":"f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c"},"spaceowner":{"opaque_id":"f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c"},"expires":"2024-01-26T13:04:31+01:00","processing":false} +{"id":"f066244d-97b2-48e7-a30d-b40fcb60cec6","space":"f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c","filename":"bar.txt","offset":0,"size":4321,"executant":{"idp":"https://cloud.ocis.test","opaque_id":"f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c"},"spaceowner":{"opaque_id":"f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c"},"expires":"2024-01-26T13:18:47+01:00","processing":false} ``` Command to clear expired uploads @@ -74,6 +94,17 @@ Cleaned uploads: - 455bd640-cd08-46e8-a5a0-9304908bd40a (Filename: file_example_PPT_1MB.ppt, Size: 1028608, Expires: 2022-08-17T12:35:34+02:00) ``` +Deprecated list command to identify unfinished uploads + +```bash +ocis storage-users uploads list +``` + +```plaintext +Incomplete uploads: + - 455bd640-cd08-46e8-a5a0-9304908bd40a (file_example_PPT_1MB.ppt, Size: 1028608, Expires: 2022-08-17T12:35:34+02:00) +``` + ### Purge Expired Space Trash-Bins Items diff --git a/services/storage-users/pkg/command/uploads.go b/services/storage-users/pkg/command/uploads.go index 7f1763785..1c651cc9f 100644 --- a/services/storage-users/pkg/command/uploads.go +++ b/services/storage-users/pkg/command/uploads.go @@ -1,12 +1,18 @@ package command import ( + "encoding/json" "fmt" "os" + "strconv" + "strings" "sync" + "time" + tw "github.com/olekukonko/tablewriter" "github.com/urfave/cli/v2" + userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" "github.com/cs3org/reva/v2/pkg/storage" "github.com/cs3org/reva/v2/pkg/storage/fs/registry" "github.com/owncloud/ocis/v2/ocis-pkg/config/configlog" @@ -22,6 +28,7 @@ func Uploads(cfg *config.Config) *cli.Command { Usage: "manage unfinished uploads", Subcommands: []*cli.Command{ ListUploads(cfg), + ListUploadSessions(cfg), PurgeExpiredUploads(cfg), }, } @@ -31,7 +38,7 @@ func Uploads(cfg *config.Config) *cli.Command { func ListUploads(cfg *config.Config) *cli.Command { return &cli.Command{ Name: "list", - Usage: "Print a list of all incomplete uploads", + Usage: "Print a list of all incomplete uploads (deprecated, use sessions)", Before: func(c *cli.Context) error { return configlog.ReturnFatal(parser.ParseConfig(cfg)) }, @@ -50,7 +57,7 @@ func ListUploads(cfg *config.Config) *cli.Command { managingFS, ok := fs.(storage.UploadSessionLister) if !ok { - fmt.Fprintf(os.Stderr, "'%s' storage does not support listing expired uploads\n", cfg.Driver) + fmt.Fprintf(os.Stderr, "'%s' storage does not support listing upload sessions\n", cfg.Driver) os.Exit(1) } expired := false @@ -69,7 +76,162 @@ func ListUploads(cfg *config.Config) *cli.Command { } } -// PurgeExpiredUploads is the entry point for the server command. +// ListUploadSessions prints a list of upload sessiens +func ListUploadSessions(cfg *config.Config) *cli.Command { + return &cli.Command{ + Name: "sessions", + Usage: "Print a list of upload sessions", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "id", + DefaultText: "unset", + Usage: "filter sessions by upload session id", + }, + &cli.BoolFlag{ + Name: "processing", + DefaultText: "unset", + Usage: "filter sessions by processing status", + }, + &cli.BoolFlag{ + Name: "expired", + DefaultText: "unset", + Usage: "filter sessions by expired status", + }, + &cli.BoolFlag{ + Name: "json", + Usage: "output as json", + }, + }, + Before: func(c *cli.Context) error { + return configlog.ReturnFatal(parser.ParseConfig(cfg)) + }, + Action: func(c *cli.Context) error { + f, ok := registry.NewFuncs[cfg.Driver] + if !ok { + fmt.Fprintf(os.Stderr, "Unknown filesystem driver '%s'\n", cfg.Driver) + os.Exit(1) + } + drivers := revaconfig.StorageProviderDrivers(cfg) + fs, err := f(drivers[cfg.Driver].(map[string]interface{}), nil) + if err != nil { + fmt.Fprintf(os.Stderr, "Failed to initialize filesystem driver '%s'\n", cfg.Driver) + return err + } + + managingFS, ok := fs.(storage.UploadSessionLister) + if !ok { + fmt.Fprintf(os.Stderr, "'%s' storage does not support listing upload sessions\n", cfg.Driver) + os.Exit(1) + } + + var b strings.Builder + filter := storage.UploadSessionFilter{} + if c.IsSet("processing") { + processingValue := c.Bool("processing") + filter.Processing = &processingValue + if !processingValue { + b.WriteString("Not ") + } + if b.Len() == 0 { + b.WriteString("Processing ") + } else { + b.WriteString("processing ") + } + } + if c.IsSet("expired") { + expiredValue := c.Bool("expired") + filter.Expired = &expiredValue + if !expiredValue { + if b.Len() == 0 { + b.WriteString("Not ") + } else { + b.WriteString(", not ") + } + } + if b.Len() == 0 { + b.WriteString("Expired ") + } else { + b.WriteString("expired ") + } + } + if b.Len() == 0 { + b.WriteString("Sessions") + } else { + b.WriteString("sessions") + } + if c.IsSet("id") { + idValue := c.String("id") + filter.ID = &idValue + b.WriteString(" with id '" + idValue + "'") + } + b.WriteString(":") + uploads, err := managingFS.ListUploadSessions(c.Context, filter) + if err != nil { + return err + } + + var table *tw.Table + if c.Bool("json") { + for _, u := range uploads { + ref := u.Reference() + s := struct { + ID string `json:"id"` + Space string `json:"space"` + Filename string `json:"filename"` + Offset int64 `json:"offset"` + Size int64 `json:"size"` + Executant userpb.UserId `json:"executant"` + SpaceOwner *userpb.UserId `json:"spaceowner,omitempty"` + Expires time.Time `json:"expires"` + Processing bool `json:"processing"` + }{ + Space: ref.GetResourceId().GetSpaceId(), + ID: u.ID(), + Filename: u.Filename(), + Offset: u.Offset(), + Size: u.Size(), + Executant: u.Executant(), + SpaceOwner: u.SpaceOwner(), + Expires: u.Expires(), + Processing: u.IsProcessing(), + } + j, err := json.Marshal(s) + if err != nil { + fmt.Println(err) + } + fmt.Println(string(j)) + } + } else { + + // Print what the user requested + fmt.Println(b.String()) + + // start a table + table = tw.NewWriter(os.Stdout) + table.SetHeader([]string{"Space", "Upload Id", "Name", "Offset", "Size", "Executant", "Owner", "Expires", "Processing"}) + table.SetAutoFormatHeaders(false) + + for _, u := range uploads { + table.Append([]string{ + u.Reference().ResourceId.GetSpaceId(), + u.ID(), + u.Filename(), + strconv.FormatInt(u.Offset(), 10), + strconv.FormatInt(u.Size(), 10), + u.Executant().OpaqueId, + u.SpaceOwner().GetOpaqueId(), + u.Expires().Format(time.RFC3339), + strconv.FormatBool(u.IsProcessing()), + }) + } + table.Render() + } + return nil + }, + } +} + +// PurgeExpiredUploads is the entry point for the clean command func PurgeExpiredUploads(cfg *config.Config) *cli.Command { return &cli.Command{ Name: "clean",