diff --git a/changelog/unreleased/remove-revisions-cli.md b/changelog/unreleased/remove-revisions-cli.md new file mode 100644 index 0000000000..15afc95510 --- /dev/null +++ b/changelog/unreleased/remove-revisions-cli.md @@ -0,0 +1,5 @@ +Enhancement: Add cli to purge revisions + +Adds a cli that allows removing all revisions for a storage-provider. + +https://github.com/owncloud/ocis/pull/9497 diff --git a/ocis/pkg/command/revisions.go b/ocis/pkg/command/revisions.go new file mode 100644 index 0000000000..7de6e3367b --- /dev/null +++ b/ocis/pkg/command/revisions.go @@ -0,0 +1,109 @@ +package command + +import ( + "errors" + "fmt" + + ocisbs "github.com/cs3org/reva/v2/pkg/storage/fs/ocis/blobstore" + s3bs "github.com/cs3org/reva/v2/pkg/storage/fs/s3ng/blobstore" + "github.com/owncloud/ocis/v2/ocis-pkg/config" + "github.com/owncloud/ocis/v2/ocis-pkg/config/configlog" + "github.com/owncloud/ocis/v2/ocis-pkg/config/parser" + "github.com/owncloud/ocis/v2/ocis/pkg/register" + "github.com/owncloud/ocis/v2/ocis/pkg/revisions" + "github.com/urfave/cli/v2" +) + +// RevisionsCommand is the entrypoint for the revisions command. +func RevisionsCommand(cfg *config.Config) *cli.Command { + return &cli.Command{ + Name: "revisions", + Usage: "ocis revisions functionality", + Subcommands: []*cli.Command{ + PurgeRevisionsCommand(cfg), + }, + Before: func(c *cli.Context) error { + return configlog.ReturnError(parser.ParseConfig(cfg, true)) + }, + Action: func(_ *cli.Context) error { + fmt.Println("Read the docs") + return nil + }, + } +} + +// PurgeRevisionsCommand allows removing all revisions from a storage provider. +func PurgeRevisionsCommand(cfg *config.Config) *cli.Command { + return &cli.Command{ + Name: "purge", + Usage: "purge all revisions", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "basepath", + Aliases: []string{"p"}, + Usage: "the basepath of the decomposedfs (e.g. /var/tmp/ocis/storage/metadata)", + Required: true, + }, + &cli.StringFlag{ + Name: "blobstore", + Aliases: []string{"b"}, + Usage: "the blobstore type. Can be (none, ocis, s3ng). Default ocis", + Value: "ocis", + }, + &cli.BoolFlag{ + Name: "dry-run", + Usage: "do not delete anything, just print what would be deleted", + Value: true, + }, + &cli.BoolFlag{ + Name: "verbose", + Aliases: []string{"v"}, + Usage: "print verbose output", + Value: false, + }, + }, + Action: func(c *cli.Context) error { + basePath := c.String("basepath") + if basePath == "" { + fmt.Println("basepath is required") + return cli.ShowCommandHelp(c, "revisions") + } + + var ( + bs revisions.DelBlobstore + err error + ) + switch c.String("blobstore") { + case "s3ng": + bs, err = s3bs.New( + cfg.StorageUsers.Drivers.S3NG.Endpoint, + cfg.StorageUsers.Drivers.S3NG.Region, + cfg.StorageUsers.Drivers.S3NG.Bucket, + cfg.StorageUsers.Drivers.S3NG.AccessKey, + cfg.StorageUsers.Drivers.S3NG.SecretKey, + s3bs.Options{}, + ) + case "ocis": + bs, err = ocisbs.New(basePath) + case "none": + bs = nil + default: + err = errors.New("blobstore type not supported") + } + if err != nil { + fmt.Println(err) + return err + } + if err := revisions.PurgeRevisions(basePath, bs, c.Bool("dry-run"), c.Bool("verbose")); err != nil { + fmt.Printf("❌ Error purging revisions: %s", err) + return err + } + + return nil + }, + } +} + +func init() { + register.AddCommand(RevisionsCommand) +} diff --git a/ocis/pkg/revisions/revisions.go b/ocis/pkg/revisions/revisions.go new file mode 100644 index 0000000000..f30eca457d --- /dev/null +++ b/ocis/pkg/revisions/revisions.go @@ -0,0 +1,131 @@ +// Package revisions allows manipulating revisions in a storage provider. +package revisions + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "regexp" + + "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/node" + "github.com/shamaton/msgpack/v2" +) + +var ( + // _nodesGlobPattern is the glob pattern to find all nodes + _nodesGlobPattern = "spaces/*/*/*/*/*/*/*/*" + // regex to determine if a node versioned. Examples: + // 9113a718-8285-4b32-9042-f930f1a58ac2.REV.2024-05-22T07:32:53.89969726Z + // 9113a718-8285-4b32-9042-f930f1a58ac2.REV.2024-05-22T07:32:53.89969726Z.mpk + // 9113a718-8285-4b32-9042-f930f1a58ac2.REV.2024-05-22T07:32:53.89969726Z.mlock + _versionRegex = regexp.MustCompile(`\.REV\.[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]+Z*`) +) + +// DelBlobstore is the interface for a blobstore that can delete blobs. +type DelBlobstore interface { + Delete(node *node.Node) error +} + +// PurgeRevisions removes all revisions from a storage provider. +func PurgeRevisions(p string, bs DelBlobstore, dryRun bool, verbose bool) error { + pattern := filepath.Join(p, _nodesGlobPattern) + if verbose { + fmt.Println("Looking for nodes in", pattern) + } + + nodes, err := filepath.Glob(pattern) + if err != nil { + return err + } + + if len(nodes) == 0 { + return errors.New("no nodes found, double check storage path") + } + + countFiles := 0 + countBlobs := 0 + countRevisions := 0 + for _, d := range nodes { + if !_versionRegex.MatchString(d) { + continue + } + + var blobID string + e := filepath.Ext(d) + switch e { + case ".mpk": + blobID, err = getBlobID(d) + if err != nil { + fmt.Printf("error getting blobID from %s: %v\n", d, err) + continue + } + + countBlobs++ + case ".mlock": + // no extra action on .mlock files + default: + countRevisions++ + } + + if !dryRun { + if blobID != "" { + // TODO: needs spaceID for s3ng + if err := bs.Delete(&node.Node{BlobID: blobID}); err != nil { + fmt.Printf("error deleting blob %s: %v\n", blobID, err) + continue + } + } + + if err := os.Remove(d); err != nil { + fmt.Printf("error removing %s: %v\n", d, err) + continue + } + + } + + countFiles++ + + if verbose { + if dryRun { + fmt.Println("Would delete", d) + if blobID != "" { + fmt.Println("Would delete blob", blobID) + } + } else { + fmt.Println("Deleted", d) + if blobID != "" { + fmt.Println("Deleted blob", blobID) + } + } + } + } + + switch { + case countFiles == 0 && countRevisions == 0 && countBlobs == 0: + fmt.Println("❎ No revisions found. Storage provider is clean.") + case !dryRun: + fmt.Printf("✅ Deleted %d revisions (%d files / %d blobs)\n", countRevisions, countFiles, countBlobs) + default: + fmt.Printf("👉 Would delete %d revisions (%d files / %d blobs)\n", countRevisions, countFiles, countBlobs) + } + return nil +} + +func getBlobID(path string) (string, error) { + b, err := os.ReadFile(path) + if err != nil { + return "", err + } + + m := map[string][]byte{} + if err := msgpack.Unmarshal(b, &m); err != nil { + return "", err + } + + if bid := m["user.ocis.blobid"]; string(bid) != "" { + return string(bid), nil + } + + return "", nil +}