[server][storage] single shared temp directory

server/internal/command/config.defaults.yml

@@ -8,7 +8,8 @@ db:
   trace: false        # Trace queries (dev)
 
 storage:
-  location: storage
+  root: storage
+  tmp: tmp            # relative to root. Must start with '/' for absolute path
   
 user:
   password:

server/internal/storage/config.go

@@ -1,5 +1,6 @@
 package storage
 
 type Config struct {
-	Location string `koanf:"location"`
+	Root string `koanf:"root"`
+	Temp string `koanf:"tmp"`
 }

server/internal/storage/drivers.go

@@ -10,7 +10,7 @@ var drivers = map[string]Driver{
 		},
 	},
 	"minio": {
-		Params: []string{"endpoint", "key_id", "access_key", "tmp", "bucket_name", "prefix"},
+		Params: []string{"endpoint", "key_id", "access_key", "bucket_name", "prefix"},
 		Create: createMinioBackend,
 	},
 }

server/internal/storage/local_storage.go

@@ -11,7 +11,6 @@ import (
 type localStorage struct {
 	name string
 	root string
-	tmp  string
 }
 
 func createLocalBackend(name string, params map[string]string) (LocalBackend, error) {
@@ -22,14 +21,10 @@ func createLocalBackend(name string, params map[string]string) (LocalBackend, er
 	l := localStorage{
 		name: name,
 		root: params["root"],
-		tmp:  filepath.Join(params["root"], "tmp"),
 	}
 
-	if err := os.RemoveAll(l.tmp); err != nil {
-		return nil, errors.New("failed to remove tmp directory: " + err.Error())
-	}
-	if err := os.MkdirAll(l.tmp, 0750); err != nil {
-		return nil, errors.New("failed to create tmp directory: " + err.Error())
+	if err := os.MkdirAll(l.root, 0700); err != nil {
+		return nil, errors.New("failed to create root directory: " + err.Error())
 	}
 
 	return l, nil
@@ -51,7 +46,7 @@ func (l localStorage) OpenRead(name string, start, length int) (io.ReadCloser, e
 }
 
 func (l localStorage) OpenWrite(name string) (io.WriteCloser, error) {
-	f, err := os.CreateTemp(l.tmp, "upload-*")
+	f, err := os.CreateTemp(tempDir, "phylum-*")
 	if err != nil {
 		return nil, err
 	}

server/internal/storage/minio_storage.go

@@ -16,30 +16,22 @@ var ErrRangeNotSupported = errors.New("byte range not suppoered")
 type minioStorage struct {
 	name       string
 	client     *minio.Client
-	tmp        string
 	bucketName string
 	prefix     string
 }
 
 func createMinioBackend(name string, params map[string]string) (Backend, error) {
 	if params["endpoint"] == "" {
-		return nil, errors.New("minio endpoint not provided")
+		return nil, errors.New("endpoint not provided")
 	}
 	if params["key_id"] == "" {
-		return nil, errors.New("minio key id not provided")
+		return nil, errors.New("key id not provided")
 	}
 	if params["access_key"] == "" {
-		return nil, errors.New("minio access key not provided")
-	}
-	if params["tmp"] == "" {
-		return nil, errors.New("minio tmp not provided")
+		return nil, errors.New("access key not provided")
 	}
 	if params["bucket_name"] == "" {
-		return nil, errors.New("minio bucket name not provided")
-	}
-	err := os.MkdirAll(params["tmp"], 0750)
-	if err != nil {
-		return nil, err
+		return nil, errors.New("bucket name not provided")
 	}
 
 	client, err := minio.New(params["endpoint"], &minio.Options{
@@ -49,7 +41,7 @@ func createMinioBackend(name string, params map[string]string) (Backend, error)
 	if err != nil {
 		return nil, err
 	}
-	return minioStorage{name: name, client: client, tmp: params["tmp"], bucketName: params["bucket_name"], prefix: params["prefix"]}, nil
+	return minioStorage{name: name, client: client, bucketName: params["bucket_name"], prefix: params["prefix"]}, nil
 }
 
 func (s minioStorage) Name() string {
@@ -73,7 +65,7 @@ func (s minioStorage) Copy(name string, src Backend) error {
 	if l, ok := src.(LocalBackend); ok {
 		localPath = l.path(name)
 	} else {
-		f, err := os.CreateTemp(s.tmp, "phylum-*")
+		f, err := os.CreateTemp(tempDir, "phylum-*")
 		if err != nil {
 			return err
 		}
@@ -130,7 +122,7 @@ func (s minioStorage) DeleteAll(names []string) []error {
 }
 
 func (s minioStorage) String() string {
-	return fmt.Sprintf("minio (endpoint: %s, bucket: %s, prefix: %s, tmp: %s)", s.client.EndpointURL(), s.bucketName, s.prefix, s.tmp)
+	return fmt.Sprintf("minio (endpoint: %s, bucket: %s, prefix: %s)", s.client.EndpointURL(), s.bucketName, s.prefix)
 }
 
 // type progressReader struct {

server/internal/storage/storage.go

@@ -6,6 +6,8 @@ import (
 	"fmt"
 	"os"
 	"path"
+	"path/filepath"
+	"strings"
 
 	"github.com/jackc/pgx/v5"
 	"github.com/shroff/phylum/server/internal/db"
@@ -18,6 +20,7 @@ const DefaultBackendName = "_"
 
 var defaultBackend LocalBackend
 var backends map[string]Backend
+var tempDir string
 
 type BackendConfig struct {
 	Name   string            `json:"name"`
@@ -26,19 +29,31 @@ type BackendConfig struct {
 }
 
 func Initialize(db db.Handler) error {
-	if err := os.MkdirAll(Cfg.Location, 0700); err != nil {
-		return errors.New("failed to create storage location(" + Cfg.Location + "): " + err.Error())
+	if err := os.MkdirAll(Cfg.Root, 0700); err != nil {
+		return errors.New("failed to create storage root(" + Cfg.Root + "): " + err.Error())
 	} else {
-		if stat, err := os.Stat(Cfg.Location); err != nil {
-			return errors.New("failed to stat storage location(" + Cfg.Location + "): " + err.Error())
+		if stat, err := os.Stat(Cfg.Root); err != nil {
+			return errors.New("failed to stat storage root(" + Cfg.Root + "): " + err.Error())
 		} else if stat.Mode()&0xfff != 0700 {
-			return errors.New("storage location does not have correct permissions (0700): " + fmt.Sprintf("%o", stat.Mode()&0xfff))
+			return errors.New("storage root does not have correct permissions (0700): " + fmt.Sprintf("%o", stat.Mode()&0xfff))
 		}
 
 	}
+
+	tempDir = filepath.Join(Cfg.Root, Cfg.Temp)
+	if strings.HasPrefix(Cfg.Temp, string(filepath.Separator)) {
+		tempDir = Cfg.Temp
+	}
+	if err := os.RemoveAll(tempDir); err != nil {
+		return errors.New("failed to clear temp directory: " + err.Error())
+	}
+	if err := os.MkdirAll(tempDir, 0700); err != nil {
+		return errors.New("failed to create temp directory: " + err.Error())
+	}
+
 	if restoredBackends, err := restoreBackends(db); err != nil {
 		return errors.New("failed to restore backends: " + err.Error())
-	} else if b, err := createLocalBackend(DefaultBackendName, map[string]string{"root": path.Join(Cfg.Location, "default")}); err != nil {
+	} else if b, err := createLocalBackend(DefaultBackendName, map[string]string{"root": path.Join(Cfg.Root, "default")}); err != nil {
 		return errors.New("failed to create default backend: " + err.Error())
 	} else {
 		backends = restoredBackends