diff --git a/ocis/pkg/revisions/revisions.go b/ocis/pkg/revisions/revisions.go index 249dbf37e1..d2d1eac686 100644 --- a/ocis/pkg/revisions/revisions.go +++ b/ocis/pkg/revisions/revisions.go @@ -2,7 +2,6 @@ package revisions import ( - "errors" "fmt" "os" "path/filepath" @@ -26,25 +25,118 @@ type DelBlobstore interface { Delete(node *node.Node) error } -// PurgeRevisions removes all revisions from a storage provider. -func PurgeRevisions(pattern string, bs DelBlobstore, dryRun bool, verbose bool) error { +// PurgeRevisionsGlob removes all revisions from a storage provider using globbing. +func PurgeRevisionsGlob(pattern string, bs DelBlobstore, dryRun bool, verbose bool) (int, int, int) { if verbose { fmt.Println("Looking for nodes in", pattern) } - nodes, err := filepath.Glob(pattern) + ch := make(chan string) + go func() { + defer close(ch) + nodes, err := filepath.Glob(pattern) + if err != nil { + fmt.Println("error globbing", pattern, err) + return + } + + if len(nodes) == 0 { + fmt.Println("no nodes found. Double check storage path") + return + } + + for _, n := range nodes { + if _versionRegex.MatchString(n) { + ch <- n + } + } + }() + + return purgeRevisions(ch, bs, dryRun, verbose) +} + +// PurgeRevisionsWalk removes all revisions from a storage provider using walking. +func PurgeRevisionsWalk(base string, bs DelBlobstore, dryRun bool, verbose bool) (int, int, int) { + ch := make(chan string) + go func() { + defer close(ch) + err := filepath.Walk(base, func(path string, info os.FileInfo, err error) error { + if err != nil { + fmt.Println("error walking", base, err) + return err + } + + if !_versionRegex.MatchString(info.Name()) { + return nil + } + + ch <- path + return nil + }) + if err != nil { + fmt.Println("error walking", base, err) + return + } + + }() + return purgeRevisions(ch, bs, dryRun, verbose) +} + +// PurgeRevisionsList removes all revisions from a storage provider using listing. +func PurgeRevisionsList(base string, bs DelBlobstore, dryRun bool, verbose bool) (int, int, int) { + ch := make(chan string) + go func() { + defer close(ch) + if err := listFolder(base, ch); err != nil { + fmt.Println("error listing", base, err) + return + } + }() + + return purgeRevisions(ch, bs, dryRun, verbose) +} + +func listFolder(path string, ch chan<- string) error { + children, err := os.ReadDir(path) if err != nil { return err } - if len(nodes) == 0 { - return errors.New("no nodes found, double check storage path") - } + for _, child := range children { + if child.IsDir() { + if err := listFolder(filepath.Join(path, child.Name()), ch); err != nil { + return err + } + } + if _versionRegex.MatchString(child.Name()) { + ch <- filepath.Join(path, child.Name()) + } + + } + return nil +} + +// PrintResults prints the results +func PrintResults(countFiles, countBlobs, countRevisions int, dryRun bool) error { + switch { + case countFiles == 0 && countRevisions == 0 && countBlobs == 0: + fmt.Println("❎ No revisions found. Storage provider is clean.") + case !dryRun: + fmt.Printf("✅ Deleted %d revisions (%d files / %d blobs)\n", countRevisions, countFiles, countBlobs) + default: + fmt.Printf("👉 Would delete %d revisions (%d files / %d blobs)\n", countRevisions, countFiles, countBlobs) + } + return nil +} + +func purgeRevisions(nodes <-chan string, bs DelBlobstore, dryRun, verbose bool) (int, int, int) { countFiles := 0 countBlobs := 0 countRevisions := 0 - for _, d := range nodes { + + var err error + for d := range nodes { if !_versionRegex.MatchString(d) { continue } @@ -106,15 +198,7 @@ func PurgeRevisions(pattern string, bs DelBlobstore, dryRun bool, verbose bool) } } - switch { - case countFiles == 0 && countRevisions == 0 && countBlobs == 0: - fmt.Println("❎ No revisions found. Storage provider is clean.") - case !dryRun: - fmt.Printf("✅ Deleted %d revisions (%d files / %d blobs)\n", countRevisions, countFiles, countBlobs) - default: - fmt.Printf("👉 Would delete %d revisions (%d files / %d blobs)\n", countRevisions, countFiles, countBlobs) - } - return nil + return countFiles, countBlobs, countRevisions } func getBlobID(path string) (string, error) { diff --git a/ocis/pkg/revisions/revisions_test.go b/ocis/pkg/revisions/revisions_test.go new file mode 100644 index 0000000000..477ccbebd9 --- /dev/null +++ b/ocis/pkg/revisions/revisions_test.go @@ -0,0 +1,138 @@ +package revisions + +import ( + "fmt" + "io/fs" + "os" + "path/filepath" + "strconv" + "testing" + + "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/lookup" + "github.com/google/uuid" + "github.com/test-go/testify/require" +) + +var ( + _basePath = "test_temp/spaces/8f/638374-6ea8-4f0d-80c4-66d9b49830a5/nodes/" +) + +// func TestInit(t *testing.T) { +// initialize(10, 2) +// defer os.RemoveAll("test_temp") +// } + +func TestGlob30(t *testing.T) { testGlob(t, 10, 2) } +func TestGlob80(t *testing.T) { testGlob(t, 20, 3) } +func TestGlob250(t *testing.T) { testGlob(t, 50, 4) } +func TestGlob600(t *testing.T) { testGlob(t, 100, 5) } + +func TestWalk30(t *testing.T) { testWalk(t, 10, 2) } +func TestWalk80(t *testing.T) { testWalk(t, 20, 3) } +func TestWalk250(t *testing.T) { testWalk(t, 50, 4) } +func TestWalk600(t *testing.T) { testWalk(t, 100, 5) } + +func TestList30(t *testing.T) { testList(t, 10, 2) } +func TestList80(t *testing.T) { testList(t, 20, 3) } +func TestList250(t *testing.T) { testList(t, 50, 4) } +func TestList600(t *testing.T) { testList(t, 100, 5) } + +func BenchmarkGlob30(b *testing.B) { benchmarkGlob(b, 10, 2) } +func BenchmarkWalk30(b *testing.B) { benchmarkWalk(b, 10, 2) } +func BenchmarkList30(b *testing.B) { benchmarkList(b, 10, 2) } + +func BenchmarkGlob80(b *testing.B) { benchmarkGlob(b, 20, 3) } +func BenchmarkWalk80(b *testing.B) { benchmarkWalk(b, 20, 3) } +func BenchmarkList80(b *testing.B) { benchmarkList(b, 20, 3) } + +func BenchmarkGlob250(b *testing.B) { benchmarkGlob(b, 50, 4) } +func BenchmarkWalk250(b *testing.B) { benchmarkWalk(b, 50, 4) } +func BenchmarkList250(b *testing.B) { benchmarkList(b, 50, 4) } + +func BenchmarkGlob600(b *testing.B) { benchmarkGlob(b, 100, 5) } +func BenchmarkWalk600(b *testing.B) { benchmarkWalk(b, 100, 5) } +func BenchmarkList600(b *testing.B) { benchmarkList(b, 100, 5) } + +func BenchmarkGlob11000(b *testing.B) { benchmarkGlob(b, 1000, 10) } +func BenchmarkWalk11000(b *testing.B) { benchmarkWalk(b, 1000, 10) } +func BenchmarkList11000(b *testing.B) { benchmarkList(b, 1000, 10) } + +func BenchmarkGlob110000(b *testing.B) { benchmarkGlob(b, 10000, 10) } +func BenchmarkWalk110000(b *testing.B) { benchmarkWalk(b, 10000, 10) } +func BenchmarkList110000(b *testing.B) { benchmarkList(b, 10000, 10) } + +func benchmarkGlob(b *testing.B, numNodes int, numRevisions int) { + initialize(numNodes, numRevisions) + defer os.RemoveAll("test_temp") + + for i := 0; i < b.N; i++ { + PurgeRevisionsGlob(_basePath+"*/*/*/*/*", nil, false, false) + } +} + +func benchmarkWalk(b *testing.B, numNodes int, numRevisions int) { + initialize(numNodes, numRevisions) + defer os.RemoveAll("test_temp") + + for i := 0; i < b.N; i++ { + PurgeRevisionsWalk(_basePath, nil, false, false) + } +} + +func benchmarkList(b *testing.B, numNodes int, numRevisions int) { + initialize(numNodes, numRevisions) + defer os.RemoveAll("test_temp") + + for i := 0; i < b.N; i++ { + PurgeRevisionsList(_basePath, nil, false, false) + } +} + +func testGlob(t *testing.T, numNodes int, numRevisions int) { + initialize(numNodes, numRevisions) + defer os.RemoveAll("test_temp") + + _, _, revisions := PurgeRevisionsGlob(_basePath+"*/*/*/*/*", nil, false, false) + require.Equal(t, numNodes*numRevisions, revisions, "Deleted Revisions") +} + +func testWalk(t *testing.T, numNodes int, numRevisions int) { + initialize(numNodes, numRevisions) + defer os.RemoveAll("test_temp") + + _, _, revisions := PurgeRevisionsWalk(_basePath, nil, false, false) + require.Equal(t, numNodes*numRevisions, revisions, "Deleted Revisions") +} + +func testList(t *testing.T, numNodes int, numRevisions int) { + initialize(numNodes, numRevisions) + defer os.RemoveAll("test_temp") + + _, _, revisions := PurgeRevisionsList(_basePath, nil, false, false) + require.Equal(t, numNodes*numRevisions, revisions, "Deleted Revisions") +} + +func initialize(numNodes int, numRevisions int) { + // create base path + if err := os.MkdirAll(_basePath, fs.ModePerm); err != nil { + fmt.Println("Error creating test_temp directory", err) + os.Exit(1) + } + + for i := 0; i < numNodes; i++ { + path := lookup.Pathify(uuid.New().String(), 4, 2) + dir := filepath.Dir(path) + if err := os.MkdirAll(_basePath+dir, fs.ModePerm); err != nil { + fmt.Println("Error creating test_temp directory", err) + os.Exit(1) + } + + if _, err := os.Create(_basePath + path); err != nil { + fmt.Println("Error creating file", err) + os.Exit(1) + } + for i := 0; i < numRevisions; i++ { + os.Create(_basePath + path + ".REV.2024-05-22T07:32:53.89969" + strconv.Itoa(i) + "Z") + } + } +}