mirror of
https://github.com/opencloud-eu/opencloud.git
synced 2026-01-04 11:19:39 -06:00
feat(ocis): add revisions purge tests and benchmarks
Signed-off-by: jkoberg <jkoberg@owncloud.com>
This commit is contained in:
jkoberg
@@ -2,7 +2,6 @@
|
||||
package revisions
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -26,25 +25,118 @@ type DelBlobstore interface {
|
||||
Delete(node *node.Node) error
|
||||
}
|
||||
|
||||
// PurgeRevisions removes all revisions from a storage provider.
|
||||
func PurgeRevisions(pattern string, bs DelBlobstore, dryRun bool, verbose bool) error {
|
||||
// PurgeRevisionsGlob removes all revisions from a storage provider using globbing.
|
||||
func PurgeRevisionsGlob(pattern string, bs DelBlobstore, dryRun bool, verbose bool) (int, int, int) {
|
||||
if verbose {
|
||||
fmt.Println("Looking for nodes in", pattern)
|
||||
}
|
||||
|
||||
nodes, err := filepath.Glob(pattern)
|
||||
ch := make(chan string)
|
||||
go func() {
|
||||
defer close(ch)
|
||||
nodes, err := filepath.Glob(pattern)
|
||||
if err != nil {
|
||||
fmt.Println("error globbing", pattern, err)
|
||||
return
|
||||
}
|
||||
|
||||
if len(nodes) == 0 {
|
||||
fmt.Println("no nodes found. Double check storage path")
|
||||
return
|
||||
}
|
||||
|
||||
for _, n := range nodes {
|
||||
if _versionRegex.MatchString(n) {
|
||||
ch <- n
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
return purgeRevisions(ch, bs, dryRun, verbose)
|
||||
}
|
||||
|
||||
// PurgeRevisionsWalk removes all revisions from a storage provider using walking.
|
||||
func PurgeRevisionsWalk(base string, bs DelBlobstore, dryRun bool, verbose bool) (int, int, int) {
|
||||
ch := make(chan string)
|
||||
go func() {
|
||||
defer close(ch)
|
||||
err := filepath.Walk(base, func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
fmt.Println("error walking", base, err)
|
||||
return err
|
||||
}
|
||||
|
||||
if !_versionRegex.MatchString(info.Name()) {
|
||||
return nil
|
||||
}
|
||||
|
||||
ch <- path
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
fmt.Println("error walking", base, err)
|
||||
return
|
||||
}
|
||||
|
||||
}()
|
||||
return purgeRevisions(ch, bs, dryRun, verbose)
|
||||
}
|
||||
|
||||
// PurgeRevisionsList removes all revisions from a storage provider using listing.
|
||||
func PurgeRevisionsList(base string, bs DelBlobstore, dryRun bool, verbose bool) (int, int, int) {
|
||||
ch := make(chan string)
|
||||
go func() {
|
||||
defer close(ch)
|
||||
if err := listFolder(base, ch); err != nil {
|
||||
fmt.Println("error listing", base, err)
|
||||
return
|
||||
}
|
||||
}()
|
||||
|
||||
return purgeRevisions(ch, bs, dryRun, verbose)
|
||||
}
|
||||
|
||||
func listFolder(path string, ch chan<- string) error {
|
||||
children, err := os.ReadDir(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if len(nodes) == 0 {
|
||||
return errors.New("no nodes found, double check storage path")
|
||||
}
|
||||
for _, child := range children {
|
||||
if child.IsDir() {
|
||||
if err := listFolder(filepath.Join(path, child.Name()), ch); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if _versionRegex.MatchString(child.Name()) {
|
||||
ch <- filepath.Join(path, child.Name())
|
||||
}
|
||||
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// PrintResults prints the results
|
||||
func PrintResults(countFiles, countBlobs, countRevisions int, dryRun bool) error {
|
||||
switch {
|
||||
case countFiles == 0 && countRevisions == 0 && countBlobs == 0:
|
||||
fmt.Println("❎ No revisions found. Storage provider is clean.")
|
||||
case !dryRun:
|
||||
fmt.Printf("✅ Deleted %d revisions (%d files / %d blobs)\n", countRevisions, countFiles, countBlobs)
|
||||
default:
|
||||
fmt.Printf("👉 Would delete %d revisions (%d files / %d blobs)\n", countRevisions, countFiles, countBlobs)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func purgeRevisions(nodes <-chan string, bs DelBlobstore, dryRun, verbose bool) (int, int, int) {
|
||||
countFiles := 0
|
||||
countBlobs := 0
|
||||
countRevisions := 0
|
||||
for _, d := range nodes {
|
||||
|
||||
var err error
|
||||
for d := range nodes {
|
||||
if !_versionRegex.MatchString(d) {
|
||||
continue
|
||||
}
|
||||
@@ -106,15 +198,7 @@ func PurgeRevisions(pattern string, bs DelBlobstore, dryRun bool, verbose bool)
|
||||
}
|
||||
}
|
||||
|
||||
switch {
|
||||
case countFiles == 0 && countRevisions == 0 && countBlobs == 0:
|
||||
fmt.Println("❎ No revisions found. Storage provider is clean.")
|
||||
case !dryRun:
|
||||
fmt.Printf("✅ Deleted %d revisions (%d files / %d blobs)\n", countRevisions, countFiles, countBlobs)
|
||||
default:
|
||||
fmt.Printf("👉 Would delete %d revisions (%d files / %d blobs)\n", countRevisions, countFiles, countBlobs)
|
||||
}
|
||||
return nil
|
||||
return countFiles, countBlobs, countRevisions
|
||||
}
|
||||
|
||||
func getBlobID(path string) (string, error) {
|
||||
|
||||
138
ocis/pkg/revisions/revisions_test.go
Normal file
138
ocis/pkg/revisions/revisions_test.go
Normal file
@@ -0,0 +1,138 @@
|
||||
package revisions
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"testing"
|
||||
|
||||
"github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/lookup"
|
||||
"github.com/google/uuid"
|
||||
"github.com/test-go/testify/require"
|
||||
)
|
||||
|
||||
var (
|
||||
_basePath = "test_temp/spaces/8f/638374-6ea8-4f0d-80c4-66d9b49830a5/nodes/"
|
||||
)
|
||||
|
||||
// func TestInit(t *testing.T) {
|
||||
// initialize(10, 2)
|
||||
// defer os.RemoveAll("test_temp")
|
||||
// }
|
||||
|
||||
func TestGlob30(t *testing.T) { testGlob(t, 10, 2) }
|
||||
func TestGlob80(t *testing.T) { testGlob(t, 20, 3) }
|
||||
func TestGlob250(t *testing.T) { testGlob(t, 50, 4) }
|
||||
func TestGlob600(t *testing.T) { testGlob(t, 100, 5) }
|
||||
|
||||
func TestWalk30(t *testing.T) { testWalk(t, 10, 2) }
|
||||
func TestWalk80(t *testing.T) { testWalk(t, 20, 3) }
|
||||
func TestWalk250(t *testing.T) { testWalk(t, 50, 4) }
|
||||
func TestWalk600(t *testing.T) { testWalk(t, 100, 5) }
|
||||
|
||||
func TestList30(t *testing.T) { testList(t, 10, 2) }
|
||||
func TestList80(t *testing.T) { testList(t, 20, 3) }
|
||||
func TestList250(t *testing.T) { testList(t, 50, 4) }
|
||||
func TestList600(t *testing.T) { testList(t, 100, 5) }
|
||||
|
||||
func BenchmarkGlob30(b *testing.B) { benchmarkGlob(b, 10, 2) }
|
||||
func BenchmarkWalk30(b *testing.B) { benchmarkWalk(b, 10, 2) }
|
||||
func BenchmarkList30(b *testing.B) { benchmarkList(b, 10, 2) }
|
||||
|
||||
func BenchmarkGlob80(b *testing.B) { benchmarkGlob(b, 20, 3) }
|
||||
func BenchmarkWalk80(b *testing.B) { benchmarkWalk(b, 20, 3) }
|
||||
func BenchmarkList80(b *testing.B) { benchmarkList(b, 20, 3) }
|
||||
|
||||
func BenchmarkGlob250(b *testing.B) { benchmarkGlob(b, 50, 4) }
|
||||
func BenchmarkWalk250(b *testing.B) { benchmarkWalk(b, 50, 4) }
|
||||
func BenchmarkList250(b *testing.B) { benchmarkList(b, 50, 4) }
|
||||
|
||||
func BenchmarkGlob600(b *testing.B) { benchmarkGlob(b, 100, 5) }
|
||||
func BenchmarkWalk600(b *testing.B) { benchmarkWalk(b, 100, 5) }
|
||||
func BenchmarkList600(b *testing.B) { benchmarkList(b, 100, 5) }
|
||||
|
||||
func BenchmarkGlob11000(b *testing.B) { benchmarkGlob(b, 1000, 10) }
|
||||
func BenchmarkWalk11000(b *testing.B) { benchmarkWalk(b, 1000, 10) }
|
||||
func BenchmarkList11000(b *testing.B) { benchmarkList(b, 1000, 10) }
|
||||
|
||||
func BenchmarkGlob110000(b *testing.B) { benchmarkGlob(b, 10000, 10) }
|
||||
func BenchmarkWalk110000(b *testing.B) { benchmarkWalk(b, 10000, 10) }
|
||||
func BenchmarkList110000(b *testing.B) { benchmarkList(b, 10000, 10) }
|
||||
|
||||
func benchmarkGlob(b *testing.B, numNodes int, numRevisions int) {
|
||||
initialize(numNodes, numRevisions)
|
||||
defer os.RemoveAll("test_temp")
|
||||
|
||||
for i := 0; i < b.N; i++ {
|
||||
PurgeRevisionsGlob(_basePath+"*/*/*/*/*", nil, false, false)
|
||||
}
|
||||
}
|
||||
|
||||
func benchmarkWalk(b *testing.B, numNodes int, numRevisions int) {
|
||||
initialize(numNodes, numRevisions)
|
||||
defer os.RemoveAll("test_temp")
|
||||
|
||||
for i := 0; i < b.N; i++ {
|
||||
PurgeRevisionsWalk(_basePath, nil, false, false)
|
||||
}
|
||||
}
|
||||
|
||||
func benchmarkList(b *testing.B, numNodes int, numRevisions int) {
|
||||
initialize(numNodes, numRevisions)
|
||||
defer os.RemoveAll("test_temp")
|
||||
|
||||
for i := 0; i < b.N; i++ {
|
||||
PurgeRevisionsList(_basePath, nil, false, false)
|
||||
}
|
||||
}
|
||||
|
||||
func testGlob(t *testing.T, numNodes int, numRevisions int) {
|
||||
initialize(numNodes, numRevisions)
|
||||
defer os.RemoveAll("test_temp")
|
||||
|
||||
_, _, revisions := PurgeRevisionsGlob(_basePath+"*/*/*/*/*", nil, false, false)
|
||||
require.Equal(t, numNodes*numRevisions, revisions, "Deleted Revisions")
|
||||
}
|
||||
|
||||
func testWalk(t *testing.T, numNodes int, numRevisions int) {
|
||||
initialize(numNodes, numRevisions)
|
||||
defer os.RemoveAll("test_temp")
|
||||
|
||||
_, _, revisions := PurgeRevisionsWalk(_basePath, nil, false, false)
|
||||
require.Equal(t, numNodes*numRevisions, revisions, "Deleted Revisions")
|
||||
}
|
||||
|
||||
func testList(t *testing.T, numNodes int, numRevisions int) {
|
||||
initialize(numNodes, numRevisions)
|
||||
defer os.RemoveAll("test_temp")
|
||||
|
||||
_, _, revisions := PurgeRevisionsList(_basePath, nil, false, false)
|
||||
require.Equal(t, numNodes*numRevisions, revisions, "Deleted Revisions")
|
||||
}
|
||||
|
||||
func initialize(numNodes int, numRevisions int) {
|
||||
// create base path
|
||||
if err := os.MkdirAll(_basePath, fs.ModePerm); err != nil {
|
||||
fmt.Println("Error creating test_temp directory", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
for i := 0; i < numNodes; i++ {
|
||||
path := lookup.Pathify(uuid.New().String(), 4, 2)
|
||||
dir := filepath.Dir(path)
|
||||
if err := os.MkdirAll(_basePath+dir, fs.ModePerm); err != nil {
|
||||
fmt.Println("Error creating test_temp directory", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
if _, err := os.Create(_basePath + path); err != nil {
|
||||
fmt.Println("Error creating file", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
for i := 0; i < numRevisions; i++ {
|
||||
os.Create(_basePath + path + ".REV.2024-05-22T07:32:53.89969" + strconv.Itoa(i) + "Z")
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user