From d7b9712f13c973cb2b1d8459992edd8030996845 Mon Sep 17 00:00:00 2001 From: Abhishek Shroff Date: Wed, 7 Aug 2024 00:20:07 +0530 Subject: [PATCH] Sequential Delete --- server/internal/app/core/filesystem.go | 2 +- server/internal/db/resources.sql.go | 17 ++++++--------- server/internal/storage/local_storage.go | 18 ++++++++-------- server/internal/storage/minio_storage.go | 15 +++++++------- server/internal/storage/storage.go | 24 ++++++++++++++++++---- server/internal/storage/storage_backend.go | 4 ++-- server/sql/queries/resources.sql | 2 +- 7 files changed, 48 insertions(+), 34 deletions(-) diff --git a/server/internal/app/core/filesystem.go b/server/internal/app/core/filesystem.go index b3fb6abe..00ce55ed 100644 --- a/server/internal/app/core/filesystem.go +++ b/server/internal/app/core/filesystem.go @@ -231,7 +231,7 @@ func (f filesystem) DeleteRecursive(r Resource, hardDelete bool) error { if err != nil { return err } - errors := f.cs.Delete(deleted) + errors := f.cs.DeleteAll(deleted) for err := range errors { logrus.Warn(err) } diff --git a/server/internal/db/resources.sql.go b/server/internal/db/resources.sql.go index e71049bc..e03563bf 100644 --- a/server/internal/db/resources.sql.go +++ b/server/internal/db/resources.sql.go @@ -82,27 +82,22 @@ WITH RECURSIVE nodes(id, parent) AS ( ) DELETE FROM resources WHERE id in (SELECT id FROM nodes) -RETURNING id, dir +RETURNING id ` -type HardDeleteRecursiveRow struct { - ID uuid.UUID - Dir bool -} - -func (q *Queries) HardDeleteRecursive(ctx context.Context, id uuid.UUID) ([]HardDeleteRecursiveRow, error) { +func (q *Queries) HardDeleteRecursive(ctx context.Context, id uuid.UUID) ([]uuid.UUID, error) { rows, err := q.db.Query(ctx, hardDeleteRecursive, id) if err != nil { return nil, err } defer rows.Close() - var items []HardDeleteRecursiveRow + var items []uuid.UUID for rows.Next() { - var i HardDeleteRecursiveRow - if err := rows.Scan(&i.ID, &i.Dir); err != nil { + var id uuid.UUID + if err := rows.Scan(&id); err != nil { return nil, err } - items = append(items, i) + items = append(items, id) } if err := rows.Err(); err != nil { return nil, err diff --git a/server/internal/storage/local_storage.go b/server/internal/storage/local_storage.go index 774a45b6..77527916 100644 --- a/server/internal/storage/local_storage.go +++ b/server/internal/storage/local_storage.go @@ -6,11 +6,11 @@ import ( "errors" "hash" "io" + "io/fs" "os" "path/filepath" "github.com/google/uuid" - "github.com/shroff/phylum/server/internal/db" ) type localStorage struct { @@ -59,14 +59,16 @@ func (l localStorage) OpenWrite(id uuid.UUID, callback func(int, string) error) }}, nil } -func (l localStorage) Delete(rows []db.HardDeleteRecursiveRow) []error { +func (l localStorage) Delete(id uuid.UUID) error { + return os.Remove(l.path(id)) +} + +func (l localStorage) DeleteAll(ids uuid.UUIDs) []error { errs := make([]error, 0) - for _, row := range rows { - if !row.Dir { - err := os.Remove(l.path(row.ID)) - if err != nil { - errs = append(errs, err) - } + for _, id := range ids { + err := l.Delete(id) + if err != nil && !errors.Is(err, fs.ErrNotExist) { + errs = append(errs, err) } } return errs diff --git a/server/internal/storage/minio_storage.go b/server/internal/storage/minio_storage.go index 5d13ce0f..e08e9793 100644 --- a/server/internal/storage/minio_storage.go +++ b/server/internal/storage/minio_storage.go @@ -14,7 +14,6 @@ import ( "github.com/google/uuid" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" - "github.com/shroff/phylum/server/internal/db" ) type minioStorage struct { @@ -90,15 +89,17 @@ func (s minioStorage) OpenWrite(id uuid.UUID, callback func(int, string) error) } -func (s minioStorage) Delete(rows []db.HardDeleteRecursiveRow) []error { +func (s minioStorage) Delete(id uuid.UUID) error { + return s.client.RemoveObject(context.Background(), s.bucketName, s.prefix+id.String(), minio.RemoveObjectOptions{}) +} + +func (s minioStorage) DeleteAll(ids uuid.UUIDs) []error { objs := make(chan minio.ObjectInfo) go func() { defer close(objs) - for _, row := range rows { - if !row.Dir { - objs <- minio.ObjectInfo{ - Key: s.prefix + row.ID.String(), - } + for _, id := range ids { + objs <- minio.ObjectInfo{ + Key: s.prefix + id.String(), } } }() diff --git a/server/internal/storage/storage.go b/server/internal/storage/storage.go index c4e524ee..c4ede113 100644 --- a/server/internal/storage/storage.go +++ b/server/internal/storage/storage.go @@ -14,7 +14,8 @@ type Storage interface { ListBackends() map[string]Backend OpenRead(id uuid.UUID, start, length int64) (io.ReadCloser, error) OpenWrite(id uuid.UUID, callback func(int, string) error) (io.WriteCloser, error) - Delete(rows []db.HardDeleteRecursiveRow) []error + Delete(id uuid.UUID) error + DeleteAll(ids uuid.UUIDs) []error } type storage struct { @@ -49,10 +50,25 @@ func (s storage) OpenWrite(id uuid.UUID, callback func(int, string) error) (io.W return backend.OpenWrite(id, callback) } } -func (s storage) Delete(rows []db.HardDeleteRecursiveRow) []error { - // TODO: implement - return nil +func (s storage) Delete(id uuid.UUID) error { + if backend, err := s.findStorageBackend(id); err != nil { + return err + } else { + return backend.Delete(id) + } +} + +func (s storage) DeleteAll(ids uuid.UUIDs) []error { + // TODO: Batch delete + errs := make([]error, 1) + for _, id := range ids { + err := s.Delete(id) + if err != nil { + errs = append(errs, err) + } + } + return errs } func (s *storage) findStorageBackend(id uuid.UUID) (Backend, error) { diff --git a/server/internal/storage/storage_backend.go b/server/internal/storage/storage_backend.go index 6af4dfd5..294864fd 100644 --- a/server/internal/storage/storage_backend.go +++ b/server/internal/storage/storage_backend.go @@ -4,13 +4,13 @@ import ( "io" "github.com/google/uuid" - "github.com/shroff/phylum/server/internal/db" ) type Backend interface { Name() string OpenRead(id uuid.UUID, start, length int64) (io.ReadCloser, error) OpenWrite(id uuid.UUID, callback func(int, string) error) (io.WriteCloser, error) - Delete(rows []db.HardDeleteRecursiveRow) []error + Delete(id uuid.UUID) error + DeleteAll(ids uuid.UUIDs) []error String() string } diff --git a/server/sql/queries/resources.sql b/server/sql/queries/resources.sql index 4d007f1d..e776e9dd 100644 --- a/server/sql/queries/resources.sql +++ b/server/sql/queries/resources.sql @@ -69,4 +69,4 @@ WITH RECURSIVE nodes(id, parent) AS ( ) DELETE FROM resources WHERE id in (SELECT id FROM nodes) -RETURNING id, dir; +RETURNING id;