[server][core] Hard Delete Expired

server/internal/command/serve/cmd.go

@@ -1,10 +1,15 @@
 package serve
 
 import (
+	"context"
+	"fmt"
+	"time"
+
 	"github.com/fvbock/endless"
 	"github.com/shroff/phylum/server/internal/api/publink"
 	apiv1 "github.com/shroff/phylum/server/internal/api/v1"
 	"github.com/shroff/phylum/server/internal/api/webdav"
+	"github.com/shroff/phylum/server/internal/core/fs"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
@@ -22,6 +27,12 @@ func SetupCommand() *cobra.Command {
 			webdav.SetupHandler(engine.Group(config.GetString("webdav_prefix")))
 			apiv1.Setup(engine.Group("/api/v1"))
 			publink.Setup(engine.Group("/publink"))
+			t := time.Now().Add(-30 * 24 * time.Hour)
+			if d, err := fs.HardDeleteExpired(context.Background(), t); err != nil {
+				logrus.Fatal(err)
+			} else {
+				logrus.Info(fmt.Sprintf("Removed %d files deleted before %s", d, t.Format(time.RFC1123)))
+			}
 
 			server := endless.NewServer(config.GetString("listen"), engine)
 			server.BeforeBegin = func(addr string) {

server/internal/core/db/trash.sql.go

@@ -61,6 +61,35 @@ func (q *Queries) HardDelete(ctx context.Context, arg HardDeleteParams) ([]HardD
 	return items, nil
 }
 
+const hardDeleteExpired = `-- name: HardDeleteExpired :many
+DELETE FROM resources WHERE deleted < $1::TIMESTAMP RETURNING id, dir
+`
+
+type HardDeleteExpiredRow struct {
+	ID  uuid.UUID
+	Dir bool
+}
+
+func (q *Queries) HardDeleteExpired(ctx context.Context, t pgtype.Timestamp) ([]HardDeleteExpiredRow, error) {
+	rows, err := q.db.Query(ctx, hardDeleteExpired, t)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var items []HardDeleteExpiredRow
+	for rows.Next() {
+		var i HardDeleteExpiredRow
+		if err := rows.Scan(&i.ID, &i.Dir); err != nil {
+			return nil, err
+		}
+		items = append(items, i)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+	return items, nil
+}
+
 const restoreDeleted = `-- name: RestoreDeleted :many
 WITH RECURSIVE nodes(id, parent, ts) AS (
     SELECT r.id, r.parent, r.deleted

server/internal/core/fs/delete.go

@@ -71,7 +71,13 @@ func (f filesystem) deleteRecursive(id, parent uuid.UUID, softDelete, preserveRo
 	})
 
 	if err == nil {
-		errors := f.cs.DeleteAll(deleted)
+		ids := make([]uuid.UUID, len(deleted))
+		for _, d := range deleted {
+			if !d.Dir {
+				ids = append(ids, d.ID)
+			}
+		}
+		errors := f.cs.DeleteAll(ids)
 		for _, err := range errors {
 			logrus.Warn(err)
 		}

server/internal/core/fs/trash.go

@@ -1,6 +1,7 @@
 package fs
 
 import (
+	"context"
 	"encoding/base64"
 	"encoding/binary"
 	"fmt"
@@ -12,6 +13,7 @@ import (
 	"github.com/jackc/pgx/v5"
 	"github.com/jackc/pgx/v5/pgtype"
 	"github.com/shroff/phylum/server/internal/core/db"
+	"github.com/shroff/phylum/server/internal/core/storage"
 )
 
 func (f filesystem) TrashList(cursor string, n int) ([]Resource, string, error) {
@@ -159,3 +161,16 @@ func (r Resource) Restore(parentPathOrUUID string, name string, autoRename bool)
 	res = r
 	return
 }
+
+func HardDeleteExpired(ctx context.Context, t time.Time) (int, error) {
+	d := db.Get()
+	rows, err := d.HardDeleteExpired(ctx, pgtype.Timestamp{Valid: true, Time: t.UTC()})
+	ids := make([]uuid.UUID, len(rows))
+	for _, d := range rows {
+		if !d.Dir {
+			ids = append(ids, d.ID)
+		}
+	}
+	storage.Get().DeleteAll(ids)
+	return len(rows), err
+}

server/internal/core/storage/storage.go

@@ -19,7 +19,7 @@ type Storage interface {
 	OpenWrite(id uuid.UUID, h func() hash.Hash, callback func(int, string, string) error) (io.WriteCloser, error)
 	CopyContents(src, dest uuid.UUID) error
 	Delete(id uuid.UUID) error
-	DeleteAll(deleted []db.HardDeleteRow) []error
+	DeleteAll(deleted []uuid.UUID) []error
 }
 
 type storage struct {
@@ -104,15 +104,13 @@ func (s storage) Delete(id uuid.UUID) error {
 	}
 }
 
-func (s storage) DeleteAll(deleted []db.HardDeleteRow) []error {
+func (s storage) DeleteAll(ids []uuid.UUID) []error {
 	// TODO: Batch delete
 	var errs []error
-	for _, d := range deleted {
-		if !d.Dir {
-			err := s.Delete(d.ID)
-			if err != nil {
-				errs = append(errs, err)
-			}
+	for _, id := range ids {
+		err := s.Delete(id)
+		if err != nil {
+			errs = append(errs, err)
 		}
 	}
 	return errs

server/sql/queries/trash.sql

@@ -35,6 +35,9 @@ DELETE FROM resources
     WHERE id IN (SELECT id FROM nodes)
     RETURNING id, dir;
 
+-- name: HardDeleteExpired :many
+DELETE FROM resources WHERE deleted < @t::TIMESTAMP RETURNING id, dir;
+
 -- name: RestoreDeleted :many
 WITH RECURSIVE nodes(id, parent, ts) AS (
     SELECT r.id, r.parent, r.deleted